From e1e44d059096dc9e9d70d029d343c2929f7201c7 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Thu, 10 Jun 2010 22:04:57 +0000 Subject: [PATCH 001/200] Starting a new branch for LUCENE-2324 and more realtime-search related patches. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@953476 13f79535-47bb-0310-9956-ffa450edef68 From 42b9f0caa32ff71782a949264d033a346ede7b26 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Wed, 21 Jul 2010 10:27:20 +0000 Subject: [PATCH 002/200] LUCENE-2324: Committing second version of the patch to the real-time branch. It's not done yet, but easier to track progress using the branch. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@966168 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/BufferedDeletes.java | 169 ------- .../lucene/index/BufferedDeletesInRAM.java | 70 +++ .../apache/lucene/index/ByteBlockPool.java | 14 +- .../apache/lucene/index/ByteSliceReader.java | 14 +- .../apache/lucene/index/ByteSliceWriter.java | 6 +- .../org/apache/lucene/index/DocConsumer.java | 5 +- .../lucene/index/DocConsumerPerThread.java | 33 -- .../apache/lucene/index/DocFieldConsumer.java | 12 +- .../index/DocFieldConsumerPerField.java | 1 + .../index/DocFieldConsumerPerThread.java | 27 -- .../lucene/index/DocFieldConsumers.java | 89 ++-- .../index/DocFieldConsumersPerField.java | 13 +- .../index/DocFieldConsumersPerThread.java | 75 --- .../lucene/index/DocFieldProcessor.java | 365 +++++++++++++- .../index/DocFieldProcessorPerField.java | 4 +- .../index/DocFieldProcessorPerThread.java | 393 --------------- .../org/apache/lucene/index/DocInverter.java | 92 ++-- .../lucene/index/DocInverterPerField.java | 37 +- .../lucene/index/DocInverterPerThread.java | 92 ---- .../index/DocumentsWriterPerThread.java | 459 ++++++++++++++++++ .../index/DocumentsWriterRAMAllocator.java | 148 ++++++ .../index/DocumentsWriterThreadPool.java | 255 ++++++++++ .../index/DocumentsWriterThreadState.java | 50 -- .../lucene/index/FreqProxFieldMergeState.java | 113 ----- .../lucene/index/FreqProxTermsWriter.java | 249 ++-------- .../index/FreqProxTermsWriterPerField.java | 143 +++++- .../index/FreqProxTermsWriterPerThread.java | 45 -- .../lucene/index/IndexWriterConfig.java | 23 +- .../org/apache/lucene/index/IntBlockPool.java | 14 +- .../lucene/index/InvertedDocConsumer.java | 14 +- .../index/InvertedDocConsumerPerThread.java | 27 -- .../lucene/index/InvertedDocEndConsumer.java | 9 +- .../InvertedDocEndConsumerPerThread.java | 25 - .../org/apache/lucene/index/NormsWriter.java | 115 ++--- .../lucene/index/NormsWriterPerField.java | 8 +- .../lucene/index/NormsWriterPerThread.java | 45 -- .../lucene/index/ParallelPostingsArray.java | 2 +- .../org/apache/lucene/index/SegmentInfo.java | 21 + .../lucene/index/StoredFieldsWriter.java | 72 ++- .../index/StoredFieldsWriterPerThread.java | 79 --- .../lucene/index/TermVectorsTermsWriter.java | 108 +++-- .../index/TermVectorsTermsWriterPerField.java | 38 +- .../TermVectorsTermsWriterPerThread.java | 89 ---- .../org/apache/lucene/index/TermsHash.java | 153 +++--- .../lucene/index/TermsHashConsumer.java | 7 +- .../index/TermsHashConsumerPerThread.java | 27 -- .../lucene/index/TermsHashPerField.java | 79 +-- ...readAffinityDocumentsWriterThreadPool.java | 66 +++ .../util/ThreadSafeCloneableSortedMap.java | 156 ++++++ .../apache/lucene/index/TestByteSlices.java | 2 +- .../apache/lucene/index/TestIndexWriter.java | 7 +- .../lucene/index/TestIndexWriterConfig.java | 69 +-- .../lucene/index/TestIndexWriterDelete.java | 5 +- .../index/TestNRTReaderWithThreads.java | 3 +- .../lucene/index/TestStressIndexing2.java | 2 +- .../lucene/index/TestThreadedOptimize.java | 3 +- 56 files changed, 2266 insertions(+), 1975 deletions(-) delete mode 100644 lucene/src/java/org/apache/lucene/index/BufferedDeletes.java create mode 100644 lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java create mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java create mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java create mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java delete mode 100644 lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java delete mode 100644 lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/NormsWriterPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/StoredFieldsWriterPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerThread.java delete mode 100644 lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerThread.java create mode 100644 lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java create mode 100644 lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java diff --git a/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java b/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java deleted file mode 100644 index 56735e16741..00000000000 --- a/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java +++ /dev/null @@ -1,169 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; -import java.util.ArrayList; -import java.util.List; -import java.util.Map.Entry; - -import org.apache.lucene.search.Query; - -/** Holds buffered deletes, by docID, term or query. We - * hold two instances of this class: one for the deletes - * prior to the last flush, the other for deletes after - * the last flush. This is so if we need to abort - * (discard all buffered docs) we can also discard the - * buffered deletes yet keep the deletes done during - * previously flushed segments. */ -class BufferedDeletes { - int numTerms; - Map terms; - Map queries = new HashMap(); - List docIDs = new ArrayList(); - long bytesUsed; - private final boolean doTermSort; - - public BufferedDeletes(boolean doTermSort) { - this.doTermSort = doTermSort; - if (doTermSort) { - terms = new TreeMap(); - } else { - terms = new HashMap(); - } - } - - // Number of documents a delete term applies to. - final static class Num { - private int num; - - Num(int num) { - this.num = num; - } - - int getNum() { - return num; - } - - void setNum(int num) { - // Only record the new number if it's greater than the - // current one. This is important because if multiple - // threads are replacing the same doc at nearly the - // same time, it's possible that one thread that got a - // higher docID is scheduled before the other - // threads. - if (num > this.num) - this.num = num; - } - } - - int size() { - // We use numTerms not terms.size() intentionally, so - // that deletes by the same term multiple times "count", - // ie if you ask to flush every 1000 deletes then even - // dup'd terms are counted towards that 1000 - return numTerms + queries.size() + docIDs.size(); - } - - void update(BufferedDeletes in) { - numTerms += in.numTerms; - bytesUsed += in.bytesUsed; - terms.putAll(in.terms); - queries.putAll(in.queries); - docIDs.addAll(in.docIDs); - in.clear(); - } - - void clear() { - terms.clear(); - queries.clear(); - docIDs.clear(); - numTerms = 0; - bytesUsed = 0; - } - - void addBytesUsed(long b) { - bytesUsed += b; - } - - boolean any() { - return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0; - } - - // Remaps all buffered deletes based on a completed - // merge - synchronized void remap(MergeDocIDRemapper mapper, - SegmentInfos infos, - int[][] docMaps, - int[] delCounts, - MergePolicy.OneMerge merge, - int mergeDocCount) { - - final Map newDeleteTerms; - - // Remap delete-by-term - if (terms.size() > 0) { - if (doTermSort) { - newDeleteTerms = new TreeMap(); - } else { - newDeleteTerms = new HashMap(); - } - for(Entry entry : terms.entrySet()) { - Num num = entry.getValue(); - newDeleteTerms.put(entry.getKey(), - new Num(mapper.remap(num.getNum()))); - } - } else - newDeleteTerms = null; - - - // Remap delete-by-docID - final List newDeleteDocIDs; - - if (docIDs.size() > 0) { - newDeleteDocIDs = new ArrayList(docIDs.size()); - for (Integer num : docIDs) { - newDeleteDocIDs.add(Integer.valueOf(mapper.remap(num.intValue()))); - } - } else - newDeleteDocIDs = null; - - - // Remap delete-by-query - final HashMap newDeleteQueries; - - if (queries.size() > 0) { - newDeleteQueries = new HashMap(queries.size()); - for(Entry entry: queries.entrySet()) { - Integer num = entry.getValue(); - newDeleteQueries.put(entry.getKey(), - Integer.valueOf(mapper.remap(num.intValue()))); - } - } else - newDeleteQueries = null; - - if (newDeleteTerms != null) - terms = newDeleteTerms; - if (newDeleteDocIDs != null) - docIDs = newDeleteDocIDs; - if (newDeleteQueries != null) - queries = newDeleteQueries; - } -} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java b/lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java new file mode 100644 index 00000000000..21ef5d52ed3 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java @@ -0,0 +1,70 @@ +package org.apache.lucene.index; + +import java.util.TreeMap; + +import org.apache.lucene.search.Query; +import org.apache.lucene.util.ThreadSafeCloneableSortedMap; + +public class BufferedDeletesInRAM { + static class Delete { + int flushCount; + + public Delete(int flushCount) { + this.flushCount = flushCount; + } + } + + final static class DeleteTerm extends Delete { + final Term term; + + public DeleteTerm(Term term, int flushCount) { + super(flushCount); + this.term = term; + } + } + + final static class DeleteTerms extends Delete { + final Term[] terms; + + public DeleteTerms(Term[] terms, int flushCount) { + super(flushCount); + this.terms = terms; + } + } + + final static class DeleteQuery extends Delete { + final Query query; + + public DeleteQuery(Query query, int flushCount) { + super(flushCount); + this.query = query; + } + } + + final ThreadSafeCloneableSortedMap deletes = ThreadSafeCloneableSortedMap + .getThreadSafeSortedMap(new TreeMap()); + + final void addDeleteTerm(Term term, long sequenceID, int numThreadStates) { + deletes.put(sequenceID, new DeleteTerm(term, numThreadStates)); + } + + final void addDeleteTerms(Term[] terms, long sequenceID, int numThreadStates) { + deletes.put(sequenceID, new DeleteTerms(terms, numThreadStates)); + } + + final void addDeleteQuery(Query query, long sequenceID, int numThreadStates) { + deletes.put(sequenceID, new DeleteQuery(query, numThreadStates)); + } + + boolean hasDeletes() { + return !deletes.isEmpty(); + } + + void clear() { + deletes.clear(); + } + + int getNumDeletes() { + return this.deletes.size(); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/ByteBlockPool.java b/lucene/src/java/org/apache/lucene/index/ByteBlockPool.java index 44701f4ba17..651c89d0ed5 100644 --- a/lucene/src/java/org/apache/lucene/index/ByteBlockPool.java +++ b/lucene/src/java/org/apache/lucene/index/ByteBlockPool.java @@ -50,10 +50,10 @@ final class ByteBlockPool { public byte[][] buffers = new byte[10][]; int bufferUpto = -1; // Which buffer we are upto - public int byteUpto = DocumentsWriter.BYTE_BLOCK_SIZE; // Where we are in head buffer + public int byteUpto = DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; // Where we are in head buffer public byte[] buffer; // Current head buffer - public int byteOffset = -DocumentsWriter.BYTE_BLOCK_SIZE; // Current head offset + public int byteOffset = -DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; // Current head offset private final Allocator allocator; @@ -95,11 +95,11 @@ final class ByteBlockPool { bufferUpto++; byteUpto = 0; - byteOffset += DocumentsWriter.BYTE_BLOCK_SIZE; + byteOffset += DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; } public int newSlice(final int size) { - if (byteUpto > DocumentsWriter.BYTE_BLOCK_SIZE-size) + if (byteUpto > DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE-size) nextBuffer(); final int upto = byteUpto; byteUpto += size; @@ -123,7 +123,7 @@ final class ByteBlockPool { final int newSize = levelSizeArray[newLevel]; // Maybe allocate another block - if (byteUpto > DocumentsWriter.BYTE_BLOCK_SIZE-newSize) + if (byteUpto > DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE-newSize) nextBuffer(); final int newUpto = byteUpto; @@ -151,8 +151,8 @@ final class ByteBlockPool { // Fill in a BytesRef from term's length & bytes encoded in // byte block final BytesRef setBytesRef(BytesRef term, int textStart) { - final byte[] bytes = term.bytes = buffers[textStart >> DocumentsWriter.BYTE_BLOCK_SHIFT]; - int pos = textStart & DocumentsWriter.BYTE_BLOCK_MASK; + final byte[] bytes = term.bytes = buffers[textStart >> DocumentsWriterRAMAllocator.BYTE_BLOCK_SHIFT]; + int pos = textStart & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; if ((bytes[pos] & 0x80) == 0) { // length is 1 byte term.length = bytes[pos]; diff --git a/lucene/src/java/org/apache/lucene/index/ByteSliceReader.java b/lucene/src/java/org/apache/lucene/index/ByteSliceReader.java index a298aa0cb33..0a500b4710f 100644 --- a/lucene/src/java/org/apache/lucene/index/ByteSliceReader.java +++ b/lucene/src/java/org/apache/lucene/index/ByteSliceReader.java @@ -48,16 +48,16 @@ final class ByteSliceReader extends DataInput { this.endIndex = endIndex; level = 0; - bufferUpto = startIndex / DocumentsWriter.BYTE_BLOCK_SIZE; - bufferOffset = bufferUpto * DocumentsWriter.BYTE_BLOCK_SIZE; + bufferUpto = startIndex / DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; + bufferOffset = bufferUpto * DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; buffer = pool.buffers[bufferUpto]; - upto = startIndex & DocumentsWriter.BYTE_BLOCK_MASK; + upto = startIndex & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; final int firstSize = ByteBlockPool.levelSizeArray[0]; if (startIndex+firstSize >= endIndex) { // There is only this one slice to read - limit = endIndex & DocumentsWriter.BYTE_BLOCK_MASK; + limit = endIndex & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; } else limit = upto+firstSize-4; } @@ -102,11 +102,11 @@ final class ByteSliceReader extends DataInput { level = ByteBlockPool.nextLevelArray[level]; final int newSize = ByteBlockPool.levelSizeArray[level]; - bufferUpto = nextIndex / DocumentsWriter.BYTE_BLOCK_SIZE; - bufferOffset = bufferUpto * DocumentsWriter.BYTE_BLOCK_SIZE; + bufferUpto = nextIndex / DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; + bufferOffset = bufferUpto * DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; buffer = pool.buffers[bufferUpto]; - upto = nextIndex & DocumentsWriter.BYTE_BLOCK_MASK; + upto = nextIndex & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; if (nextIndex + newSize >= endIndex) { // We are advancing to the final slice diff --git a/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java b/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java index a8e4d7ffa2d..ea0a8fd7d88 100644 --- a/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java +++ b/lucene/src/java/org/apache/lucene/index/ByteSliceWriter.java @@ -42,9 +42,9 @@ final class ByteSliceWriter extends DataOutput { * Set up the writer to write at address. */ public void init(int address) { - slice = pool.buffers[address >> DocumentsWriter.BYTE_BLOCK_SHIFT]; + slice = pool.buffers[address >> DocumentsWriterRAMAllocator.BYTE_BLOCK_SHIFT]; assert slice != null; - upto = address & DocumentsWriter.BYTE_BLOCK_MASK; + upto = address & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; offset0 = address; assert upto < slice.length; } @@ -80,6 +80,6 @@ final class ByteSliceWriter extends DataOutput { } public int getAddress() { - return upto + (offset0 & DocumentsWriter.BYTE_BLOCK_NOT_MASK); + return upto + (offset0 & DocumentsWriterRAMAllocator.BYTE_BLOCK_NOT_MASK); } } \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/DocConsumer.java b/lucene/src/java/org/apache/lucene/index/DocConsumer.java index d6119aa9d7d..92cb23a6c60 100644 --- a/lucene/src/java/org/apache/lucene/index/DocConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/DocConsumer.java @@ -18,11 +18,10 @@ package org.apache.lucene.index; */ import java.io.IOException; -import java.util.Collection; abstract class DocConsumer { - abstract DocConsumerPerThread addThread(DocumentsWriterThreadState perThread) throws IOException; - abstract void flush(final Collection threads, final SegmentWriteState state) throws IOException; + abstract DocumentsWriterPerThread.DocWriter processDocument() throws IOException; + abstract void flush(final SegmentWriteState state) throws IOException; abstract void closeDocStore(final SegmentWriteState state) throws IOException; abstract void abort(); abstract boolean freeRAM(); diff --git a/lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java b/lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java deleted file mode 100644 index 23a0305c8b2..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -abstract class DocConsumerPerThread { - - /** Process the document. If there is - * something for this document to be done in docID order, - * you should encapsulate that as a - * DocumentsWriter.DocWriter and return it. - * DocumentsWriter then calls finish() on this object - * when it's its turn. */ - abstract DocumentsWriter.DocWriter processDocument() throws IOException; - - abstract void abort(); -} diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java index 7588504c897..d74de0862b4 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java @@ -18,7 +18,6 @@ package org.apache.lucene.index; */ import java.io.IOException; -import java.util.Collection; import java.util.Map; abstract class DocFieldConsumer { @@ -27,7 +26,7 @@ abstract class DocFieldConsumer { /** Called when DocumentsWriter decides to create a new * segment */ - abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; + abstract void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException; /** Called when DocumentsWriter decides to close the doc * stores */ @@ -36,14 +35,17 @@ abstract class DocFieldConsumer { /** Called when an aborting exception is hit */ abstract void abort(); - /** Add a new thread */ - abstract DocFieldConsumerPerThread addThread(DocFieldProcessorPerThread docFieldProcessorPerThread) throws IOException; - /** Called when DocumentsWriter is using too much RAM. * The consumer should free RAM, if possible, returning * true if any RAM was in fact freed. */ abstract boolean freeRAM(); + + abstract void startDocument() throws IOException; + abstract DocFieldConsumerPerField addField(FieldInfo fi); + + abstract DocumentsWriterPerThread.DocWriter finishDocument() throws IOException; + void setFieldInfos(FieldInfos fieldInfos) { this.fieldInfos = fieldInfos; } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java index f70e815d8d5..960ea59eae8 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java @@ -24,4 +24,5 @@ abstract class DocFieldConsumerPerField { /** Processes all occurrences of a single field */ abstract void processFields(Fieldable[] fields, int count) throws IOException; abstract void abort(); + abstract FieldInfo getFieldInfo(); } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerThread.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerThread.java deleted file mode 100644 index c8bc1641f44..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerThread.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -abstract class DocFieldConsumerPerThread { - abstract void startDocument() throws IOException; - abstract DocumentsWriter.DocWriter finishDocument() throws IOException; - abstract DocFieldConsumerPerField addField(FieldInfo fi); - abstract void abort(); -} diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java index 50a6ceac575..36241ee02ad 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java @@ -17,12 +17,9 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.util.HashMap; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; -import java.util.HashSet; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; @@ -33,10 +30,12 @@ import org.apache.lucene.util.RamUsageEstimator; final class DocFieldConsumers extends DocFieldConsumer { final DocFieldConsumer one; final DocFieldConsumer two; + final DocumentsWriterPerThread.DocState docState; - public DocFieldConsumers(DocFieldConsumer one, DocFieldConsumer two) { + public DocFieldConsumers(DocFieldProcessor processor, DocFieldConsumer one, DocFieldConsumer two) { this.one = one; this.two = two; + this.docState = processor.docState; } @Override @@ -47,33 +46,19 @@ final class DocFieldConsumers extends DocFieldConsumer { } @Override - public void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException { + public void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException { - Map> oneThreadsAndFields = new HashMap>(); - Map> twoThreadsAndFields = new HashMap>(); + Map oneFieldsToFlush = new HashMap(); + Map twoFieldsToFlush = new HashMap(); - for (Map.Entry> entry : threadsAndFields.entrySet()) { - - final DocFieldConsumersPerThread perThread = (DocFieldConsumersPerThread) entry.getKey(); - - final Collection fields = entry.getValue(); - - Iterator fieldsIt = fields.iterator(); - Collection oneFields = new HashSet(); - Collection twoFields = new HashSet(); - while(fieldsIt.hasNext()) { - DocFieldConsumersPerField perField = (DocFieldConsumersPerField) fieldsIt.next(); - oneFields.add(perField.one); - twoFields.add(perField.two); - } - - oneThreadsAndFields.put(perThread.one, oneFields); - twoThreadsAndFields.put(perThread.two, twoFields); + for (Map.Entry fieldToFlush : fieldsToFlush.entrySet()) { + DocFieldConsumersPerField perField = (DocFieldConsumersPerField) fieldToFlush.getValue(); + oneFieldsToFlush.put(fieldToFlush.getKey(), perField.one); + twoFieldsToFlush.put(fieldToFlush.getKey(), perField.two); } - - one.flush(oneThreadsAndFields, state); - two.flush(twoThreadsAndFields, state); + one.flush(oneFieldsToFlush, state); + two.flush(twoFieldsToFlush, state); } @Override @@ -101,16 +86,11 @@ final class DocFieldConsumers extends DocFieldConsumer { return any; } - @Override - public DocFieldConsumerPerThread addThread(DocFieldProcessorPerThread docFieldProcessorPerThread) throws IOException { - return new DocFieldConsumersPerThread(docFieldProcessorPerThread, this, one.addThread(docFieldProcessorPerThread), two.addThread(docFieldProcessorPerThread)); - } - PerDoc[] docFreeList = new PerDoc[1]; int freeCount; int allocCount; - synchronized PerDoc getPerDoc() { + PerDoc getPerDoc() { if (freeCount == 0) { allocCount++; if (allocCount > docFreeList.length) { @@ -125,15 +105,15 @@ final class DocFieldConsumers extends DocFieldConsumer { return docFreeList[--freeCount]; } - synchronized void freePerDoc(PerDoc perDoc) { + void freePerDoc(PerDoc perDoc) { assert freeCount < docFreeList.length; docFreeList[freeCount++] = perDoc; } - class PerDoc extends DocumentsWriter.DocWriter { + class PerDoc extends DocumentsWriterPerThread.DocWriter { - DocumentsWriter.DocWriter writerOne; - DocumentsWriter.DocWriter writerTwo; + DocumentsWriterPerThread.DocWriter writerOne; + DocumentsWriterPerThread.DocWriter writerTwo; @Override public long sizeInBytes() { @@ -166,4 +146,35 @@ final class DocFieldConsumers extends DocFieldConsumer { } } } + + @Override + public DocumentsWriterPerThread.DocWriter finishDocument() throws IOException { + final DocumentsWriterPerThread.DocWriter oneDoc = one.finishDocument(); + final DocumentsWriterPerThread.DocWriter twoDoc = two.finishDocument(); + if (oneDoc == null) + return twoDoc; + else if (twoDoc == null) + return oneDoc; + else { + DocFieldConsumers.PerDoc both = getPerDoc(); + both.docID = docState.docID; + assert oneDoc.docID == docState.docID; + assert twoDoc.docID == docState.docID; + both.writerOne = oneDoc; + both.writerTwo = twoDoc; + return both; + } + } + + @Override + public void startDocument() throws IOException { + one.startDocument(); + two.startDocument(); + } + + @Override + public DocFieldConsumerPerField addField(FieldInfo fi) { + return new DocFieldConsumersPerField(this, fi, one.addField(fi), two.addField(fi)); + } + } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java index e75891f5556..5abf003d5a1 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java @@ -24,12 +24,14 @@ final class DocFieldConsumersPerField extends DocFieldConsumerPerField { final DocFieldConsumerPerField one; final DocFieldConsumerPerField two; - final DocFieldConsumersPerThread perThread; + final DocFieldConsumers parent; + final FieldInfo fieldInfo; - public DocFieldConsumersPerField(DocFieldConsumersPerThread perThread, DocFieldConsumerPerField one, DocFieldConsumerPerField two) { - this.perThread = perThread; + public DocFieldConsumersPerField(DocFieldConsumers parent, FieldInfo fi, DocFieldConsumerPerField one, DocFieldConsumerPerField two) { + this.parent = parent; this.one = one; this.two = two; + this.fieldInfo = fi; } @Override @@ -46,4 +48,9 @@ final class DocFieldConsumersPerField extends DocFieldConsumerPerField { two.abort(); } } + + @Override + FieldInfo getFieldInfo() { + return fieldInfo; + } } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerThread.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerThread.java deleted file mode 100644 index 99d56ee725d..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerThread.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -final class DocFieldConsumersPerThread extends DocFieldConsumerPerThread { - - final DocFieldConsumerPerThread one; - final DocFieldConsumerPerThread two; - final DocFieldConsumers parent; - final DocumentsWriter.DocState docState; - - public DocFieldConsumersPerThread(DocFieldProcessorPerThread docFieldProcessorPerThread, - DocFieldConsumers parent, DocFieldConsumerPerThread one, DocFieldConsumerPerThread two) { - this.parent = parent; - this.one = one; - this.two = two; - docState = docFieldProcessorPerThread.docState; - } - - @Override - public void startDocument() throws IOException { - one.startDocument(); - two.startDocument(); - } - - @Override - public void abort() { - try { - one.abort(); - } finally { - two.abort(); - } - } - - @Override - public DocumentsWriter.DocWriter finishDocument() throws IOException { - final DocumentsWriter.DocWriter oneDoc = one.finishDocument(); - final DocumentsWriter.DocWriter twoDoc = two.finishDocument(); - if (oneDoc == null) - return twoDoc; - else if (twoDoc == null) - return oneDoc; - else { - DocFieldConsumers.PerDoc both = parent.getPerDoc(); - both.docID = docState.docID; - assert oneDoc.docID == docState.docID; - assert twoDoc.docID == docState.docID; - both.writerOne = oneDoc; - both.writerTwo = twoDoc; - return both; - } - } - - @Override - public DocFieldConsumerPerField addField(FieldInfo fi) { - return new DocFieldConsumersPerField(this, one.addField(fi), two.addField(fi)); - } -} diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java index c2a586a6ca2..8ce56b834c4 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java @@ -19,8 +19,15 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Collection; -import java.util.Map; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.RamUsageEstimator; /** @@ -33,13 +40,27 @@ import java.util.HashMap; final class DocFieldProcessor extends DocConsumer { - final DocumentsWriter docWriter; final FieldInfos fieldInfos = new FieldInfos(); final DocFieldConsumer consumer; final StoredFieldsWriter fieldsWriter; - public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer) { - this.docWriter = docWriter; + // Holds all fields seen in current doc + DocFieldProcessorPerField[] fields = new DocFieldProcessorPerField[1]; + int fieldCount; + + // Hash table for all fields ever seen + DocFieldProcessorPerField[] fieldHash = new DocFieldProcessorPerField[2]; + int hashMask = 1; + int totalFieldCount; + + + float docBoost; + int fieldGen; + final DocumentsWriterPerThread.DocState docState; + + + public DocFieldProcessor(DocumentsWriterPerThread docWriter, DocFieldConsumer consumer) { + this.docState = docWriter.docState; this.consumer = consumer; consumer.setFieldInfos(fieldInfos); fieldsWriter = new StoredFieldsWriter(docWriter, fieldInfos); @@ -52,16 +73,17 @@ final class DocFieldProcessor extends DocConsumer { } @Override - public void flush(Collection threads, SegmentWriteState state) throws IOException { + public void flush(SegmentWriteState state) throws IOException { - Map> childThreadsAndFields = new HashMap>(); - for ( DocConsumerPerThread thread : threads) { - DocFieldProcessorPerThread perThread = (DocFieldProcessorPerThread) thread; - childThreadsAndFields.put(perThread.consumer, perThread.fields()); - perThread.trimFields(state); + Map childFields = new HashMap(); + Collection fields = fields(); + for (DocFieldConsumerPerField f : fields) { + childFields.put(f.getFieldInfo(), f); } + trimFields(state); + fieldsWriter.flush(state); - consumer.flush(childThreadsAndFields, state); + consumer.flush(childFields, state); // Important to save after asking consumer to flush so // consumer can alter the FieldInfo* if necessary. EG, @@ -74,6 +96,15 @@ final class DocFieldProcessor extends DocConsumer { @Override public void abort() { + for(int i=0;i fields() { + Collection fields = new HashSet(); + for(int i=0;i= fieldHash.length/2) + rehash(); + } else { + fp.fieldInfo.update(field.isIndexed(), field.isTermVectorStored(), + field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), + field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); + } + + if (thisFieldGen != fp.lastGen) { + + // First time we're seeing this field for this doc + fp.fieldCount = 0; + + if (fieldCount == fields.length) { + final int newSize = fields.length*2; + DocFieldProcessorPerField newArray[] = new DocFieldProcessorPerField[newSize]; + System.arraycopy(fields, 0, newArray, 0, fieldCount); + fields = newArray; + } + + fields[fieldCount++] = fp; + fp.lastGen = thisFieldGen; + } + + if (fp.fieldCount == fp.fields.length) { + Fieldable[] newArray = new Fieldable[fp.fields.length*2]; + System.arraycopy(fp.fields, 0, newArray, 0, fp.fieldCount); + fp.fields = newArray; + } + + fp.fields[fp.fieldCount++] = field; + if (field.isStored()) { + fieldsWriter.addField(field, fp.fieldInfo); + } + } + + // If we are writing vectors then we must visit + // fields in sorted order so they are written in + // sorted order. TODO: we actually only need to + // sort the subset of fields that have vectors + // enabled; we could save [small amount of] CPU + // here. + quickSort(fields, 0, fieldCount-1); + + for(int i=0;i= hi) + return; + else if (hi == 1+lo) { + if (array[lo].fieldInfo.name.compareTo(array[hi].fieldInfo.name) > 0) { + final DocFieldProcessorPerField tmp = array[lo]; + array[lo] = array[hi]; + array[hi] = tmp; + } + return; + } + + int mid = (lo + hi) >>> 1; + + if (array[lo].fieldInfo.name.compareTo(array[mid].fieldInfo.name) > 0) { + DocFieldProcessorPerField tmp = array[lo]; + array[lo] = array[mid]; + array[mid] = tmp; + } + + if (array[mid].fieldInfo.name.compareTo(array[hi].fieldInfo.name) > 0) { + DocFieldProcessorPerField tmp = array[mid]; + array[mid] = array[hi]; + array[hi] = tmp; + + if (array[lo].fieldInfo.name.compareTo(array[mid].fieldInfo.name) > 0) { + DocFieldProcessorPerField tmp2 = array[lo]; + array[lo] = array[mid]; + array[mid] = tmp2; + } + } + + int left = lo + 1; + int right = hi - 1; + + if (left >= right) + return; + + DocFieldProcessorPerField partition = array[mid]; + + for (; ;) { + while (array[right].fieldInfo.name.compareTo(partition.fieldInfo.name) > 0) + --right; + + while (left < right && array[left].fieldInfo.name.compareTo(partition.fieldInfo.name) <= 0) + ++left; + + if (left < right) { + DocFieldProcessorPerField tmp = array[left]; + array[left] = array[right]; + array[right] = tmp; + --right; + } else { + break; + } + } + + quickSort(array, lo, left); + quickSort(array, left + 1, hi); + } + + PerDoc[] docFreeList = new PerDoc[1]; + int freeCount; + int allocCount; + + PerDoc getPerDoc() { + if (freeCount == 0) { + allocCount++; + if (allocCount > docFreeList.length) { + // Grow our free list up front to make sure we have + // enough space to recycle all outstanding PerDoc + // instances + assert allocCount == 1+docFreeList.length; + docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + } + return new PerDoc(); + } else + return docFreeList[--freeCount]; + } + + void freePerDoc(PerDoc perDoc) { + assert freeCount < docFreeList.length; + docFreeList[freeCount++] = perDoc; + } + + class PerDoc extends DocumentsWriterPerThread.DocWriter { + + DocumentsWriterPerThread.DocWriter one; + DocumentsWriterPerThread.DocWriter two; + + @Override + public long sizeInBytes() { + return one.sizeInBytes() + two.sizeInBytes(); + } + + @Override + public void finish() throws IOException { + try { + try { + one.finish(); + } finally { + two.finish(); + } + } finally { + freePerDoc(this); + } + } + + @Override + public void abort() { + try { + try { + one.abort(); + } finally { + two.abort(); + } + } finally { + freePerDoc(this); + } + } } } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java b/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java index 8fb1da45280..4e961efc444 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java @@ -34,8 +34,8 @@ final class DocFieldProcessorPerField { int fieldCount; Fieldable[] fields = new Fieldable[1]; - public DocFieldProcessorPerField(final DocFieldProcessorPerThread perThread, final FieldInfo fieldInfo) { - this.consumer = perThread.consumer.addField(fieldInfo); + public DocFieldProcessorPerField(final DocFieldProcessor docFieldProcessor, final FieldInfo fieldInfo) { + this.consumer = docFieldProcessor.consumer.addField(fieldInfo); this.fieldInfo = fieldInfo; } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java b/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java deleted file mode 100644 index 51e46201c50..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java +++ /dev/null @@ -1,393 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.io.IOException; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Fieldable; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.RamUsageEstimator; - -/** - * Gathers all Fieldables for a document under the same - * name, updates FieldInfos, and calls per-field consumers - * to process field by field. - * - * Currently, only a single thread visits the fields, - * sequentially, for processing. - */ - -final class DocFieldProcessorPerThread extends DocConsumerPerThread { - - float docBoost; - int fieldGen; - final DocFieldProcessor docFieldProcessor; - final FieldInfos fieldInfos; - final DocFieldConsumerPerThread consumer; - - // Holds all fields seen in current doc - DocFieldProcessorPerField[] fields = new DocFieldProcessorPerField[1]; - int fieldCount; - - // Hash table for all fields ever seen - DocFieldProcessorPerField[] fieldHash = new DocFieldProcessorPerField[2]; - int hashMask = 1; - int totalFieldCount; - - final StoredFieldsWriterPerThread fieldsWriter; - - final DocumentsWriter.DocState docState; - - public DocFieldProcessorPerThread(DocumentsWriterThreadState threadState, DocFieldProcessor docFieldProcessor) throws IOException { - this.docState = threadState.docState; - this.docFieldProcessor = docFieldProcessor; - this.fieldInfos = docFieldProcessor.fieldInfos; - this.consumer = docFieldProcessor.consumer.addThread(this); - fieldsWriter = docFieldProcessor.fieldsWriter.addThread(docState); - } - - @Override - public void abort() { - for(int i=0;i fields() { - Collection fields = new HashSet(); - for(int i=0;i docFields = doc.getFields(); - final int numDocFields = docFields.size(); - - // Absorb any new fields first seen in this document. - // Also absorb any changes to fields we had already - // seen before (eg suddenly turning on norms or - // vectors, etc.): - - for(int i=0;i= fieldHash.length/2) - rehash(); - } else - fp.fieldInfo.update(field.isIndexed(), field.isTermVectorStored(), - field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), - field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); - - if (thisFieldGen != fp.lastGen) { - - // First time we're seeing this field for this doc - fp.fieldCount = 0; - - if (fieldCount == fields.length) { - final int newSize = fields.length*2; - DocFieldProcessorPerField newArray[] = new DocFieldProcessorPerField[newSize]; - System.arraycopy(fields, 0, newArray, 0, fieldCount); - fields = newArray; - } - - fields[fieldCount++] = fp; - fp.lastGen = thisFieldGen; - } - - if (fp.fieldCount == fp.fields.length) { - Fieldable[] newArray = new Fieldable[fp.fields.length*2]; - System.arraycopy(fp.fields, 0, newArray, 0, fp.fieldCount); - fp.fields = newArray; - } - - fp.fields[fp.fieldCount++] = field; - if (field.isStored()) { - fieldsWriter.addField(field, fp.fieldInfo); - } - } - - // If we are writing vectors then we must visit - // fields in sorted order so they are written in - // sorted order. TODO: we actually only need to - // sort the subset of fields that have vectors - // enabled; we could save [small amount of] CPU - // here. - quickSort(fields, 0, fieldCount-1); - - for(int i=0;i= hi) - return; - else if (hi == 1+lo) { - if (array[lo].fieldInfo.name.compareTo(array[hi].fieldInfo.name) > 0) { - final DocFieldProcessorPerField tmp = array[lo]; - array[lo] = array[hi]; - array[hi] = tmp; - } - return; - } - - int mid = (lo + hi) >>> 1; - - if (array[lo].fieldInfo.name.compareTo(array[mid].fieldInfo.name) > 0) { - DocFieldProcessorPerField tmp = array[lo]; - array[lo] = array[mid]; - array[mid] = tmp; - } - - if (array[mid].fieldInfo.name.compareTo(array[hi].fieldInfo.name) > 0) { - DocFieldProcessorPerField tmp = array[mid]; - array[mid] = array[hi]; - array[hi] = tmp; - - if (array[lo].fieldInfo.name.compareTo(array[mid].fieldInfo.name) > 0) { - DocFieldProcessorPerField tmp2 = array[lo]; - array[lo] = array[mid]; - array[mid] = tmp2; - } - } - - int left = lo + 1; - int right = hi - 1; - - if (left >= right) - return; - - DocFieldProcessorPerField partition = array[mid]; - - for (; ;) { - while (array[right].fieldInfo.name.compareTo(partition.fieldInfo.name) > 0) - --right; - - while (left < right && array[left].fieldInfo.name.compareTo(partition.fieldInfo.name) <= 0) - ++left; - - if (left < right) { - DocFieldProcessorPerField tmp = array[left]; - array[left] = array[right]; - array[right] = tmp; - --right; - } else { - break; - } - } - - quickSort(array, lo, left); - quickSort(array, left + 1, hi); - } - - PerDoc[] docFreeList = new PerDoc[1]; - int freeCount; - int allocCount; - - synchronized PerDoc getPerDoc() { - if (freeCount == 0) { - allocCount++; - if (allocCount > docFreeList.length) { - // Grow our free list up front to make sure we have - // enough space to recycle all outstanding PerDoc - // instances - assert allocCount == 1+docFreeList.length; - docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - } - return new PerDoc(); - } else - return docFreeList[--freeCount]; - } - - synchronized void freePerDoc(PerDoc perDoc) { - assert freeCount < docFreeList.length; - docFreeList[freeCount++] = perDoc; - } - - class PerDoc extends DocumentsWriter.DocWriter { - - DocumentsWriter.DocWriter one; - DocumentsWriter.DocWriter two; - - @Override - public long sizeInBytes() { - return one.sizeInBytes() + two.sizeInBytes(); - } - - @Override - public void finish() throws IOException { - try { - try { - one.finish(); - } finally { - two.finish(); - } - } finally { - freePerDoc(this); - } - } - - @Override - public void abort() { - try { - try { - one.abort(); - } finally { - two.abort(); - } - } finally { - freePerDoc(this); - } - } - } -} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/DocInverter.java b/lucene/src/java/org/apache/lucene/index/DocInverter.java index 35968ba120c..f34e234f163 100644 --- a/lucene/src/java/org/apache/lucene/index/DocInverter.java +++ b/lucene/src/java/org/apache/lucene/index/DocInverter.java @@ -18,12 +18,13 @@ package org.apache.lucene.index; */ import java.io.IOException; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; - import java.util.Map; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.util.AttributeSource; + /** This is a DocFieldConsumer that inverts each field, * separately, from a Document, and accepts a @@ -34,7 +35,32 @@ final class DocInverter extends DocFieldConsumer { final InvertedDocConsumer consumer; final InvertedDocEndConsumer endConsumer; - public DocInverter(InvertedDocConsumer consumer, InvertedDocEndConsumer endConsumer) { + final DocumentsWriterPerThread.DocState docState; + + final FieldInvertState fieldState = new FieldInvertState(); + + final SingleTokenAttributeSource singleToken = new SingleTokenAttributeSource(); + + static class SingleTokenAttributeSource extends AttributeSource { + final CharTermAttribute termAttribute; + final OffsetAttribute offsetAttribute; + + private SingleTokenAttributeSource() { + termAttribute = addAttribute(CharTermAttribute.class); + offsetAttribute = addAttribute(OffsetAttribute.class); + } + + public void reinit(String stringValue, int startOffset, int endOffset) { + termAttribute.setEmpty().append(stringValue); + offsetAttribute.setOffset(startOffset, endOffset); + } + } + + // Used to read a string value for a field + final ReusableStringReader stringReader = new ReusableStringReader(); + + public DocInverter(DocumentsWriterPerThread.DocState docState, InvertedDocConsumer consumer, InvertedDocEndConsumer endConsumer) { + this.docState = docState; this.consumer = consumer; this.endConsumer = endConsumer; } @@ -47,31 +73,35 @@ final class DocInverter extends DocFieldConsumer { } @Override - void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException { + void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException { - Map> childThreadsAndFields = new HashMap>(); - Map> endChildThreadsAndFields = new HashMap>(); + Map childFieldsToFlush = new HashMap(); + Map endChildFieldsToFlush = new HashMap(); - for (Map.Entry> entry : threadsAndFields.entrySet() ) { - - - DocInverterPerThread perThread = (DocInverterPerThread) entry.getKey(); - - Collection childFields = new HashSet(); - Collection endChildFields = new HashSet(); - for (final DocFieldConsumerPerField field: entry.getValue() ) { - DocInverterPerField perField = (DocInverterPerField) field; - childFields.add(perField.consumer); - endChildFields.add(perField.endConsumer); - } - - childThreadsAndFields.put(perThread.consumer, childFields); - endChildThreadsAndFields.put(perThread.endConsumer, endChildFields); + for (Map.Entry fieldToFlush : fieldsToFlush.entrySet()) { + DocInverterPerField perField = (DocInverterPerField) fieldToFlush.getValue(); + childFieldsToFlush.put(fieldToFlush.getKey(), perField.consumer); + endChildFieldsToFlush.put(fieldToFlush.getKey(), perField.endConsumer); } - consumer.flush(childThreadsAndFields, state); - endConsumer.flush(endChildThreadsAndFields, state); + consumer.flush(childFieldsToFlush, state); + endConsumer.flush(endChildFieldsToFlush, state); } + + @Override + public void startDocument() throws IOException { + consumer.startDocument(); + endConsumer.startDocument(); + } + + @Override + public DocumentsWriterPerThread.DocWriter finishDocument() throws IOException { + // TODO: allow endConsumer.finishDocument to also return + // a DocWriter + endConsumer.finishDocument(); + return consumer.finishDocument(); + } + @Override public void closeDocStore(SegmentWriteState state) throws IOException { @@ -81,17 +111,21 @@ final class DocInverter extends DocFieldConsumer { @Override void abort() { - consumer.abort(); - endConsumer.abort(); + try { + consumer.abort(); + } finally { + endConsumer.abort(); + } } @Override public boolean freeRAM() { return consumer.freeRAM(); } - + @Override - public DocFieldConsumerPerThread addThread(DocFieldProcessorPerThread docFieldProcessorPerThread) { - return new DocInverterPerThread(docFieldProcessorPerThread, this); + public DocFieldConsumerPerField addField(FieldInfo fi) { + return new DocInverterPerField(this, fi); } + } diff --git a/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java b/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java index 41d4db34cfd..09847ad4a69 100644 --- a/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java +++ b/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java @@ -35,20 +35,20 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; final class DocInverterPerField extends DocFieldConsumerPerField { - final private DocInverterPerThread perThread; - final private FieldInfo fieldInfo; + final private DocInverter parent; + final FieldInfo fieldInfo; final InvertedDocConsumerPerField consumer; final InvertedDocEndConsumerPerField endConsumer; - final DocumentsWriter.DocState docState; + final DocumentsWriterPerThread.DocState docState; final FieldInvertState fieldState; - public DocInverterPerField(DocInverterPerThread perThread, FieldInfo fieldInfo) { - this.perThread = perThread; + public DocInverterPerField(DocInverter parent, FieldInfo fieldInfo) { + this.parent = parent; this.fieldInfo = fieldInfo; - docState = perThread.docState; - fieldState = perThread.fieldState; - this.consumer = perThread.consumer.addField(this, fieldInfo); - this.endConsumer = perThread.endConsumer.addField(this, fieldInfo); + docState = parent.docState; + fieldState = parent.fieldState; + this.consumer = parent.consumer.addField(this, fieldInfo); + this.endConsumer = parent.endConsumer.addField(this, fieldInfo); } @Override @@ -84,8 +84,8 @@ final class DocInverterPerField extends DocFieldConsumerPerField { if (!field.isTokenized()) { // un-tokenized field String stringValue = field.stringValue(); final int valueLength = stringValue.length(); - perThread.singleToken.reinit(stringValue, 0, valueLength); - fieldState.attributeSource = perThread.singleToken; + parent.singleToken.reinit(stringValue, 0, valueLength); + fieldState.attributeSource = parent.singleToken; consumer.start(field); boolean success = false; @@ -93,8 +93,9 @@ final class DocInverterPerField extends DocFieldConsumerPerField { consumer.add(); success = true; } finally { - if (!success) + if (!success) { docState.docWriter.setAborting(); + } } fieldState.offset += valueLength; fieldState.length++; @@ -119,8 +120,8 @@ final class DocInverterPerField extends DocFieldConsumerPerField { if (stringValue == null) { throw new IllegalArgumentException("field must have either TokenStream, String or Reader value"); } - perThread.stringReader.init(stringValue); - reader = perThread.stringReader; + parent.stringReader.init(stringValue); + reader = parent.stringReader; } // Tokenize field and add to postingTable @@ -173,8 +174,9 @@ final class DocInverterPerField extends DocFieldConsumerPerField { consumer.add(); success = true; } finally { - if (!success) + if (!success) { docState.docWriter.setAborting(); + } } fieldState.position++; if (++fieldState.length >= maxFieldLength) { @@ -208,4 +210,9 @@ final class DocInverterPerField extends DocFieldConsumerPerField { consumer.finish(); endConsumer.finish(); } + + @Override + FieldInfo getFieldInfo() { + return this.fieldInfo; + } } diff --git a/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java deleted file mode 100644 index 2816519f9b2..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.AttributeSource; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; - -/** This is a DocFieldConsumer that inverts each field, - * separately, from a Document, and accepts a - * InvertedTermsConsumer to process those terms. */ - -final class DocInverterPerThread extends DocFieldConsumerPerThread { - final DocInverter docInverter; - final InvertedDocConsumerPerThread consumer; - final InvertedDocEndConsumerPerThread endConsumer; - final SingleTokenAttributeSource singleToken = new SingleTokenAttributeSource(); - - static class SingleTokenAttributeSource extends AttributeSource { - final CharTermAttribute termAttribute; - final OffsetAttribute offsetAttribute; - - private SingleTokenAttributeSource() { - termAttribute = addAttribute(CharTermAttribute.class); - offsetAttribute = addAttribute(OffsetAttribute.class); - } - - public void reinit(String stringValue, int startOffset, int endOffset) { - termAttribute.setEmpty().append(stringValue); - offsetAttribute.setOffset(startOffset, endOffset); - } - } - - final DocumentsWriter.DocState docState; - - final FieldInvertState fieldState = new FieldInvertState(); - - // Used to read a string value for a field - final ReusableStringReader stringReader = new ReusableStringReader(); - - public DocInverterPerThread(DocFieldProcessorPerThread docFieldProcessorPerThread, DocInverter docInverter) { - this.docInverter = docInverter; - docState = docFieldProcessorPerThread.docState; - consumer = docInverter.consumer.addThread(this); - endConsumer = docInverter.endConsumer.addThread(this); - } - - @Override - public void startDocument() throws IOException { - consumer.startDocument(); - endConsumer.startDocument(); - } - - @Override - public DocumentsWriter.DocWriter finishDocument() throws IOException { - // TODO: allow endConsumer.finishDocument to also return - // a DocWriter - endConsumer.finishDocument(); - return consumer.finishDocument(); - } - - @Override - void abort() { - try { - consumer.abort(); - } finally { - endConsumer.abort(); - } - } - - @Override - public DocFieldConsumerPerField addField(FieldInfo fi) { - return new DocInverterPerField(this, fi); - } -} diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java new file mode 100644 index 00000000000..fafd80b4289 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -0,0 +1,459 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMFile; +import org.apache.lucene.util.ArrayUtil; + +public class DocumentsWriterPerThread { + + /** + * The IndexingChain must define the {@link #getChain(DocumentsWriter)} method + * which returns the DocConsumer that the DocumentsWriter calls to process the + * documents. + */ + abstract static class IndexingChain { + abstract DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread); + } + + + static final IndexingChain defaultIndexingChain = new IndexingChain() { + + @Override + DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread) { + /* + This is the current indexing chain: + + DocConsumer / DocConsumerPerThread + --> code: DocFieldProcessor / DocFieldProcessorPerThread + --> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField + --> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField + --> code: DocInverter / DocInverterPerThread / DocInverterPerField + --> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField + --> code: TermsHash / TermsHashPerThread / TermsHashPerField + --> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField + --> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField + --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField + --> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField + --> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField + --> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField + */ + + // Build up indexing chain: + + final TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriterPerThread); + final TermsHashConsumer freqProxWriter = new FreqProxTermsWriter(); + + final InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, + new TermsHash(documentsWriterPerThread, termVectorsWriter, null)); + final NormsWriter normsWriter = new NormsWriter(); + final DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter); + return new DocFieldProcessor(documentsWriterPerThread, docInverter); + } + }; + + static class DocState { + final DocumentsWriterPerThread docWriter; + Analyzer analyzer; + int maxFieldLength; + PrintStream infoStream; + Similarity similarity; + int docID; + Document doc; + String maxTermPrefix; + + DocState(DocumentsWriterPerThread docWriter) { + this.docWriter = docWriter; + } + + // Only called by asserts + public boolean testPoint(String name) { + return docWriter.writer.testPoint(name); + } + } + + /** Called if we hit an exception at a bad time (when + * updating the index files) and must discard all + * currently buffered docs. This resets our state, + * discarding any docs added since last flush. */ + void abort() throws IOException { + try { + if (infoStream != null) { + message("docWriter: now abort"); + } + try { + consumer.abort(); + } catch (Throwable t) { + } + + docStoreSegment = null; + numDocsInStore = 0; + docStoreOffset = 0; + + // Reset all postings data + doAfterFlush(); + + } finally { + aborting = false; + if (infoStream != null) { + message("docWriter: done abort"); + } + } + } + + + final DocumentsWriterRAMAllocator ramAllocator = new DocumentsWriterRAMAllocator(); + + final DocumentsWriter parent; + final IndexWriter writer; + + final Directory directory; + final DocState docState; + final DocConsumer consumer; + private DocFieldProcessor docFieldProcessor; + + String segment; // Current segment we are working on + private String docStoreSegment; // Current doc-store segment we are writing + private int docStoreOffset; // Current starting doc-store offset of current segment + boolean aborting; // True if an abort is pending + + private final PrintStream infoStream; + private int numDocsInRAM; + private int numDocsInStore; + private int flushedDocCount; + SegmentWriteState flushState; + + long[] sequenceIDs = new long[8]; + + final List closedFiles = new ArrayList(); + + long numBytesUsed; + + public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent, IndexingChain indexingChain) { + this.directory = directory; + this.parent = parent; + this.writer = parent.indexWriter; + this.infoStream = parent.indexWriter.getInfoStream(); + this.docState = new DocState(this); + this.docState.similarity = parent.config.getSimilarity(); + this.docState.maxFieldLength = parent.config.getMaxFieldLength(); + + consumer = indexingChain.getChain(this); + if (consumer instanceof DocFieldProcessor) { + docFieldProcessor = (DocFieldProcessor) consumer; + } + + } + + void setAborting() { + aborting = true; + } + + public void addDocument(Document doc, Analyzer analyzer) throws IOException { + docState.doc = doc; + docState.analyzer = analyzer; + docState.docID = numDocsInRAM; + initSegmentName(false); + + final DocWriter perDoc; + + boolean success = false; + try { + perDoc = consumer.processDocument(); + + success = true; + } finally { + if (!success) { + if (!aborting) { + // mark document as deleted + commitDocument(-1); + } + } + } + + success = false; + try { + if (perDoc != null) { + perDoc.finish(); + } + + success = true; + } finally { + if (!success) { + setAborting(); + } + } + + } + + public void commitDocument(long sequenceID) { + if (numDocsInRAM == sequenceIDs.length) { + sequenceIDs = ArrayUtil.grow(sequenceIDs); + } + + sequenceIDs[numDocsInRAM] = sequenceID; + numDocsInRAM++; + numDocsInStore++; + } + + int getNumDocsInRAM() { + return numDocsInRAM; + } + + long getMinSequenceID() { + if (numDocsInRAM == 0) { + return -1; + } + return sequenceIDs[0]; + } + + /** Returns true if any of the fields in the current + * buffered docs have omitTermFreqAndPositions==false */ + boolean hasProx() { + return (docFieldProcessor != null) ? docFieldProcessor.fieldInfos.hasProx() + : true; + } + + Codec getCodec() { + return flushState.codec; + } + + void initSegmentName(boolean onlyDocStore) { + if (segment == null && (!onlyDocStore || docStoreSegment == null)) { + // this call is synchronized on IndexWriter.segmentInfos + segment = writer.newSegmentName(); + assert numDocsInRAM == 0; + } + if (docStoreSegment == null) { + docStoreSegment = segment; + assert numDocsInStore == 0; + } + } + + + private void initFlushState(boolean onlyDocStore) { + initSegmentName(onlyDocStore); + flushState = new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos, + docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval(), + writer.codecs); + } + + /** Reset after a flush */ + private void doAfterFlush() throws IOException { + segment = null; + numDocsInRAM = 0; + } + + /** Flush all pending docs to a new segment */ + SegmentInfo flush(boolean closeDocStore) throws IOException { + assert numDocsInRAM > 0; + + initFlushState(closeDocStore); + + docStoreOffset = numDocsInStore; + + if (infoStream != null) { + message("flush postings as segment " + flushState.segmentName + " numDocs=" + numDocsInRAM); + } + + boolean success = false; + + try { + + if (closeDocStore) { + assert flushState.docStoreSegmentName != null; + assert flushState.docStoreSegmentName.equals(flushState.segmentName); + closeDocStore(); + flushState.numDocsInStore = 0; + } + + consumer.flush(flushState); + + if (infoStream != null) { + SegmentInfo si = new SegmentInfo(flushState.segmentName, + flushState.numDocs, + directory, false, + docStoreOffset, flushState.docStoreSegmentName, + false, + hasProx(), + getCodec()); + + final long newSegmentSize = si.sizeInBytes(); + String message = " ramUsed=" + ramAllocator.nf.format(((double) numBytesUsed)/1024./1024.) + " MB" + + " newFlushedSize=" + newSegmentSize + + " docs/MB=" + ramAllocator.nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) + + " new/old=" + ramAllocator.nf.format(100.0*newSegmentSize/numBytesUsed) + "%"; + message(message); + } + + flushedDocCount += flushState.numDocs; + + long maxSequenceID = sequenceIDs[numDocsInRAM-1]; + doAfterFlush(); + + // Create new SegmentInfo, but do not add to our + // segmentInfos until deletes are flushed + // successfully. + SegmentInfo newSegment = new SegmentInfo(flushState.segmentName, + flushState.numDocs, + directory, false, + docStoreOffset, flushState.docStoreSegmentName, + false, + hasProx(), + getCodec()); + + + newSegment.setMinSequenceID(sequenceIDs[0]); + newSegment.setMaxSequenceID(maxSequenceID); + + IndexWriter.setDiagnostics(newSegment, "flush"); + success = true; + + return newSegment; + } finally { + if (!success) { + setAborting(); + } + } + } + + /** Closes the current open doc stores an returns the doc + * store segment name. This returns null if there are * + * no buffered documents. */ + String closeDocStore() throws IOException { + + // nocommit +// if (infoStream != null) +// message("closeDocStore: " + openFiles.size() + " files to flush to segment " + docStoreSegment + " numDocs=" + numDocsInStore); + + boolean success = false; + + try { + initFlushState(true); + closedFiles.clear(); + + consumer.closeDocStore(flushState); + // nocommit + //assert 0 == openFiles.size(); + + String s = docStoreSegment; + docStoreSegment = null; + docStoreOffset = 0; + numDocsInStore = 0; + success = true; + return s; + } finally { + if (!success) { + parent.abort(); + } + } + } + + + /** Get current segment name we are writing. */ + String getSegment() { + return segment; + } + + /** Returns the current doc store segment we are writing + * to. */ + String getDocStoreSegment() { + return docStoreSegment; + } + + /** Returns the doc offset into the shared doc store for + * the current buffered docs. */ + int getDocStoreOffset() { + return docStoreOffset; + } + + + @SuppressWarnings("unchecked") + List closedFiles() { + return (List) ((ArrayList) closedFiles).clone(); + } + + void addOpenFile(String name) { + synchronized(parent.openFiles) { + assert !parent.openFiles.contains(name); + parent.openFiles.add(name); + } + } + + void removeOpenFile(String name) { + synchronized(parent.openFiles) { + assert parent.openFiles.contains(name); + parent.openFiles.remove(name); + } + closedFiles.add(name); + } + + /** Consumer returns this on each doc. This holds any + * state that must be flushed synchronized "in docID + * order". We gather these and flush them in order. */ + abstract static class DocWriter { + DocWriter next; + int docID; + abstract void finish() throws IOException; + abstract void abort(); + abstract long sizeInBytes(); + + void setNext(DocWriter next) { + this.next = next; + } + } + + /** + * Create and return a new DocWriterBuffer. + */ + PerDocBuffer newPerDocBuffer() { + return new PerDocBuffer(); + } + + /** + * RAMFile buffer for DocWriters. + */ + class PerDocBuffer extends RAMFile { + + /** + * Allocate bytes used from shared pool. + */ + protected byte[] newBuffer(int size) { + assert size == DocumentsWriterRAMAllocator.PER_DOC_BLOCK_SIZE; + return ramAllocator.perDocAllocator.getByteBlock(); + } + + /** + * Recycle the bytes used. + */ + synchronized void recycle() { + if (buffers.size() > 0) { + setLength(0); + + // Recycle the blocks + ramAllocator.perDocAllocator.recycleByteBlocks(buffers); + buffers.clear(); + sizeInBytes = 0; + + assert numBuffers() == 0; + } + } + } + + void bytesUsed(long numBytes) { + ramAllocator.bytesUsed(numBytes); + } + + void message(String message) { + if (infoStream != null) + writer.message("DW: " + message); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java new file mode 100644 index 00000000000..9c7329f8ec4 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java @@ -0,0 +1,148 @@ +package org.apache.lucene.index; + +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.util.Constants; + +class DocumentsWriterRAMAllocator { + final ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE); + final ByteBlockAllocator perDocAllocator = new ByteBlockAllocator(PER_DOC_BLOCK_SIZE); + + + class ByteBlockAllocator extends ByteBlockPool.Allocator { + final int blockSize; + + ByteBlockAllocator(int blockSize) { + this.blockSize = blockSize; + } + + ArrayList freeByteBlocks = new ArrayList(); + + /* Allocate another byte[] from the shared pool */ + @Override + byte[] getByteBlock() { + final int size = freeByteBlocks.size(); + final byte[] b; + if (0 == size) { + b = new byte[blockSize]; + // Always record a block allocated, even if + // trackAllocations is false. This is necessary + // because this block will be shared between + // things that don't track allocations (term + // vectors) and things that do (freq/prox + // postings). + numBytesUsed += blockSize; + } else + b = freeByteBlocks.remove(size-1); + return b; + } + + /* Return byte[]'s to the pool */ + @Override + void recycleByteBlocks(byte[][] blocks, int start, int end) { + for(int i=start;i blocks) { + final int size = blocks.size(); + for(int i=0;i freeIntBlocks = new ArrayList(); + + /* Allocate another int[] from the shared pool */ + int[] getIntBlock() { + final int size = freeIntBlocks.size(); + final int[] b; + if (0 == size) { + b = new int[INT_BLOCK_SIZE]; + // Always record a block allocated, even if + // trackAllocations is false. This is necessary + // because this block will be shared between + // things that don't track allocations (term + // vectors) and things that do (freq/prox + // postings). + numBytesUsed += INT_BLOCK_SIZE*INT_NUM_BYTE; + } else + b = freeIntBlocks.remove(size-1); + return b; + } + + void bytesUsed(long numBytes) { + numBytesUsed += numBytes; + } + + /* Return int[]s to the pool */ + void recycleIntBlocks(int[][] blocks, int start, int end) { + for(int i=start;i. Say list + allocates ~2X size (2*POINTER). Integer is OBJ_HEADER + + int */ + final static int BYTES_PER_DEL_DOCID = 2*POINTER_NUM_BYTE + OBJECT_HEADER_BYTES + INT_NUM_BYTE; + + /* Rough logic: HashMap has an array[Entry] w/ varying + load factor (say 2 * POINTER). Entry is object w/ + Query key, Integer val, int hash, Entry next + (OBJ_HEADER + 3*POINTER + INT). Query we often + undercount (say 24 bytes). Integer is OBJ_HEADER + INT. */ + final static int BYTES_PER_DEL_QUERY = 5*POINTER_NUM_BYTE + 2*OBJECT_HEADER_BYTES + 2*INT_NUM_BYTE + 24; + + /* Initial chunks size of the shared byte[] blocks used to + store postings data */ + final static int BYTE_BLOCK_SHIFT = 15; + final static int BYTE_BLOCK_SIZE = 1 << BYTE_BLOCK_SHIFT; + final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1; + final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK; + + final static int MAX_TERM_LENGTH_UTF8 = BYTE_BLOCK_SIZE-2; + + /* Initial chunks size of the shared int[] blocks used to + store postings data */ + final static int INT_BLOCK_SHIFT = 13; + final static int INT_BLOCK_SIZE = 1 << INT_BLOCK_SHIFT; + final static int INT_BLOCK_MASK = INT_BLOCK_SIZE - 1; + + String toMB(long v) { + return nf.format(v/1024./1024.); + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java new file mode 100644 index 00000000000..f915cfbe322 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java @@ -0,0 +1,255 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Iterator; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.document.Document; +import org.apache.lucene.util.ThreadInterruptedException; + +abstract class DocumentsWriterThreadPool { + public static abstract class Task { + private boolean clearThreadBindings = false; + + protected void clearThreadBindings() { + this.clearThreadBindings = true; + } + + boolean doClearThreadBindings() { + return clearThreadBindings; + } + } + + public static abstract class PerThreadTask extends Task { + abstract T process(final DocumentsWriterPerThread perThread) throws IOException; + } + + public static abstract class AllThreadsTask extends Task { + abstract T process(final Iterator threadsIterator) throws IOException; + } + + protected abstract static class ThreadState { + private DocumentsWriterPerThread perThread; + private boolean isIdle = true; + + void start() {/* extension hook */} + void finish() {/* extension hook */} + } + + private int pauseThreads = 0; + + protected final int maxNumThreadStates; + protected ThreadState[] allThreadStates = new ThreadState[0]; + + private final Lock lock = new ReentrantLock(); + private final Condition threadStateAvailable = lock.newCondition(); + private boolean globalLock; + private boolean aborting; + + DocumentsWriterThreadPool(int maxNumThreadStates) { + this.maxNumThreadStates = (maxNumThreadStates < 1) ? IndexWriterConfig.DEFAULT_MAX_THREAD_STATES : maxNumThreadStates; + } + + public final int getMaxThreadStates() { + return this.maxNumThreadStates; + } + + void pauseAllThreads() { + lock.lock(); + try { + pauseThreads++; + while(!allThreadsIdle()) { + try { + threadStateAvailable.await(); + } catch (InterruptedException ie) { + throw new ThreadInterruptedException(ie); + } + } + } finally { + lock.unlock(); + } + } + + void resumeAllThreads() { + lock.lock(); + try { + pauseThreads--; + assert pauseThreads >= 0; + if (0 == pauseThreads) { + threadStateAvailable.signalAll(); + } + } finally { + lock.unlock(); + } + } + + private boolean allThreadsIdle() { + for (ThreadState state : allThreadStates) { + if (!state.isIdle) { + return false; + } + } + + return true; + } + + void abort() throws IOException { + pauseAllThreads(); + aborting = true; + for (ThreadState state : allThreadStates) { + state.perThread.abort(); + } + } + + void finishAbort() { + aborting = false; + resumeAllThreads(); + } + + public T executeAllThreads(AllThreadsTask task) throws IOException { + T result = null; + + lock.lock(); + try { + try { + while (globalLock) { + threadStateAvailable.await(); + } + } catch (InterruptedException ie) { + throw new ThreadInterruptedException(ie); + } + + globalLock = true; + pauseAllThreads(); + } finally { + lock.unlock(); + } + + + // all threads are idle now + + try { + final ThreadState[] localAllThreads = allThreadStates; + + result = task.process(new Iterator() { + int i = 0; + + @Override + public boolean hasNext() { + return i < localAllThreads.length; + } + + @Override + public DocumentsWriterPerThread next() { + return localAllThreads[i++].perThread; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove() not supported."); + } + }); + return result; + } finally { + lock.lock(); + try { + try { + if (task.doClearThreadBindings()) { + clearAllThreadBindings(); + } + } finally { + globalLock = false; + resumeAllThreads(); + threadStateAvailable.signalAll(); + } + } finally { + lock.unlock(); + } + + } + } + + + public final T executePerThread(DocumentsWriter documentsWriter, Document doc, PerThreadTask task) throws IOException { + ThreadState state = acquireThreadState(documentsWriter, doc); + boolean success = false; + try { + T result = task.process(state.perThread); + success = true; + return result; + } finally { + boolean abort = false; + if (!success && state.perThread.aborting) { + state.perThread.aborting = false; + abort = true; + } + + returnDocumentsWriterPerThread(state, task.doClearThreadBindings()); + + if (abort) { + documentsWriter.abort(); + } + } + } + + protected final T addNewThreadState(DocumentsWriter documentsWriter, T threadState) { + // Just create a new "private" thread state + ThreadState[] newArray = new ThreadState[1+allThreadStates.length]; + if (allThreadStates.length > 0) + System.arraycopy(allThreadStates, 0, newArray, 0, allThreadStates.length); + threadState.perThread = documentsWriter.newDocumentsWriterPerThread(); + newArray[allThreadStates.length] = threadState; + + allThreadStates = newArray; + return threadState; + } + + protected abstract ThreadState selectThreadState(Thread requestingThread, DocumentsWriter documentsWriter, Document doc); + protected void clearThreadBindings(ThreadState flushedThread) { + // subclasses can optionally override this to cleanup after a thread flushed + } + + protected void clearAllThreadBindings() { + // subclasses can optionally override this to cleanup after a thread flushed + } + + + private final ThreadState acquireThreadState(DocumentsWriter documentsWriter, Document doc) { + lock.lock(); + try { + ThreadState threadState = selectThreadState(Thread.currentThread(), documentsWriter, doc); + + try { + while (!threadState.isIdle || globalLock || aborting) { + threadStateAvailable.await(); + } + } catch (InterruptedException ie) { + throw new ThreadInterruptedException(ie); + } + + threadState.isIdle = false; + threadState.start(); + + return threadState; + + } finally { + lock.unlock(); + } + } + + private final void returnDocumentsWriterPerThread(ThreadState state, boolean clearThreadBindings) { + lock.lock(); + try { + state.finish(); + if (clearThreadBindings) { + clearThreadBindings(state); + } + state.isIdle = true; + threadStateAvailable.signalAll(); + } finally { + lock.unlock(); + } + } +} diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java deleted file mode 100644 index c60768b1490..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadState.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -/** Used by DocumentsWriter to maintain per-thread state. - * We keep a separate Posting hash and other state for each - * thread and then merge postings hashes from all threads - * when writing the segment. */ -final class DocumentsWriterThreadState { - - boolean isIdle = true; // false if this is currently in use by a thread - int numThreads = 1; // Number of threads that share this instance - boolean doFlushAfter; // true if we should flush after processing current doc - final DocConsumerPerThread consumer; - final DocumentsWriter.DocState docState; - - final DocumentsWriter docWriter; - - public DocumentsWriterThreadState(DocumentsWriter docWriter) throws IOException { - this.docWriter = docWriter; - docState = new DocumentsWriter.DocState(); - docState.maxFieldLength = docWriter.maxFieldLength; - docState.infoStream = docWriter.infoStream; - docState.similarity = docWriter.similarity; - docState.docWriter = docWriter; - consumer = docWriter.consumer.addThread(this); - } - - void doAfterFlush() { - numThreads = 0; - doFlushAfter = false; - } -} diff --git a/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java b/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java deleted file mode 100644 index 533af28a3cc..00000000000 --- a/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; -import org.apache.lucene.util.BytesRef; - -import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray; - -// TODO FI: some of this is "generic" to TermsHash* so we -// should factor it out so other consumers don't have to -// duplicate this code - -/** Used by DocumentsWriter to merge the postings from - * multiple ThreadStates when creating a segment */ -final class FreqProxFieldMergeState { - - final FreqProxTermsWriterPerField field; - final int numPostings; - private final ByteBlockPool bytePool; - final int[] termIDs; - final FreqProxPostingsArray postings; - int currentTermID; - - final BytesRef text = new BytesRef(); - - private int postingUpto = -1; - - final ByteSliceReader freq = new ByteSliceReader(); - final ByteSliceReader prox = new ByteSliceReader(); - - int docID; - int termFreq; - - public FreqProxFieldMergeState(FreqProxTermsWriterPerField field, Comparator termComp) { - this.field = field; - this.numPostings = field.termsHashPerField.numPostings; - this.bytePool = field.perThread.termsHashPerThread.bytePool; - this.termIDs = field.termsHashPerField.sortPostings(termComp); - this.postings = (FreqProxPostingsArray) field.termsHashPerField.postingsArray; - } - - boolean nextTerm() throws IOException { - postingUpto++; - if (postingUpto == numPostings) { - return false; - } - - currentTermID = termIDs[postingUpto]; - docID = 0; - - // Get BytesRef - final int textStart = postings.textStarts[currentTermID]; - bytePool.setBytesRef(text, textStart); - - field.termsHashPerField.initReader(freq, currentTermID, 0); - if (!field.fieldInfo.omitTermFreqAndPositions) { - field.termsHashPerField.initReader(prox, currentTermID, 1); - } - - // Should always be true - boolean result = nextDoc(); - assert result; - - return true; - } - - public boolean nextDoc() throws IOException { - if (freq.eof()) { - if (postings.lastDocCodes[currentTermID] != -1) { - // Return last doc - docID = postings.lastDocIDs[currentTermID]; - if (!field.omitTermFreqAndPositions) - termFreq = postings.docFreqs[currentTermID]; - postings.lastDocCodes[currentTermID] = -1; - return true; - } else - // EOF - return false; - } - - final int code = freq.readVInt(); - if (field.omitTermFreqAndPositions) - docID += code; - else { - docID += code >>> 1; - if ((code & 1) != 0) - termFreq = 1; - else - termFreq = freq.readVInt(); - } - - assert docID != postings.lastDocIDs[currentTermID]; - - return true; - } -} diff --git a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java index 7a3a062426a..dc59ab80423 100644 --- a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java @@ -19,67 +19,54 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.Iterator; +import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Comparator; -import org.apache.lucene.index.codecs.PostingsConsumer; +import org.apache.lucene.index.DocumentsWriterPerThread.DocWriter; import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermsConsumer; import org.apache.lucene.util.BytesRef; final class FreqProxTermsWriter extends TermsHashConsumer { - @Override - public TermsHashConsumerPerThread addThread(TermsHashPerThread perThread) { - return new FreqProxTermsWriterPerThread(perThread); - } - @Override void closeDocStore(SegmentWriteState state) {} @Override void abort() {} - private int flushedDocCount; - // TODO: would be nice to factor out more of this, eg the // FreqProxFieldMergeState, and code to visit all Fields // under the same FieldInfo together, up into TermsHash*. // Other writers would presumably share alot of this... @Override - public void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException { + public void flush(Map fieldsToFlush, final SegmentWriteState state) throws IOException { // Gather all FieldData's that have postings, across all // ThreadStates List allFields = new ArrayList(); - - flushedDocCount = state.numDocs; - for (Map.Entry> entry : threadsAndFields.entrySet()) { - - Collection fields = entry.getValue(); - - - for (final TermsHashConsumerPerField i : fields) { - final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) i; - if (perField.termsHashPerField.numPostings > 0) + for (TermsHashConsumerPerField f : fieldsToFlush.values()) { + final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) f; + if (perField.termsHashPerField.numPostings > 0) { allFields.add(perField); - } + } } final int numAllFields = allFields.size(); - - // Sort by field name + + // sort by field name Collections.sort(allFields); // TODO: allow Lucene user to customize this codec: final FieldsConsumer consumer = state.codec.fieldsConsumer(state); + TermsHash termsHash = null; + /* Current writer chain: FieldsConsumer @@ -92,208 +79,44 @@ final class FreqProxTermsWriter extends TermsHashConsumer { -> IMPL: FormatPostingsPositionsWriter */ - int start = 0; - while(start < numAllFields) { - final FieldInfo fieldInfo = allFields.get(start).fieldInfo; - final String fieldName = fieldInfo.name; - - int end = start+1; - while(end < numAllFields && allFields.get(end).fieldInfo.name.equals(fieldName)) - end++; + for (int fieldNumber = 0; fieldNumber < numAllFields; fieldNumber++) { + final FieldInfo fieldInfo = allFields.get(fieldNumber).fieldInfo; - FreqProxTermsWriterPerField[] fields = new FreqProxTermsWriterPerField[end-start]; - for(int i=start;i> entry : threadsAndFields.entrySet()) { - FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread) entry.getKey(); - perThread.termsHashPerThread.reset(true); + if (termsHash != null) { + termsHash.reset(); } consumer.close(); } BytesRef payload; - /* Walk through all unique text tokens (Posting - * instances) found in this field and serialize them - * into a single RAM segment. */ - void appendPostings(FreqProxTermsWriterPerField[] fields, - FieldsConsumer consumer) - throws CorruptIndexException, IOException { + @Override + public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo) { + return new FreqProxTermsWriterPerField(termsHashPerField, this, fieldInfo); + } - int numFields = fields.length; + @Override + DocWriter finishDocument() throws IOException { + return null; + } - final BytesRef text = new BytesRef(); - - final FreqProxFieldMergeState[] mergeStates = new FreqProxFieldMergeState[numFields]; - - final TermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo); - final Comparator termComp = termsConsumer.getComparator(); - - for(int i=0;imaxThreadStates will be set to * {@link #DEFAULT_MAX_THREAD_STATES}. */ - public IndexWriterConfig setMaxThreadStates(int maxThreadStates) { - this.maxThreadStates = maxThreadStates < 1 ? DEFAULT_MAX_THREAD_STATES : maxThreadStates; + public IndexWriterConfig setIndexerThreadPool(DocumentsWriterThreadPool threadPool) { + this.indexerThreadPool = threadPool; return this; } + public DocumentsWriterThreadPool getIndexerThreadPool() { + return this.indexerThreadPool; + } + /** Returns the max number of simultaneous threads that * may be indexing documents at once in IndexWriter. */ public int getMaxThreadStates() { - return maxThreadStates; + return indexerThreadPool.getMaxThreadStates(); } /** By default, IndexWriter does not pool the @@ -580,7 +584,7 @@ public final class IndexWriterConfig implements Cloneable { /** Expert: sets the {@link DocConsumer} chain to be used to process documents. */ IndexWriterConfig setIndexingChain(IndexingChain indexingChain) { - this.indexingChain = indexingChain == null ? DocumentsWriter.defaultIndexingChain : indexingChain; + this.indexingChain = indexingChain == null ? DocumentsWriterPerThread.defaultIndexingChain : indexingChain; return this; } @@ -626,7 +630,8 @@ public final class IndexWriterConfig implements Cloneable { sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n"); sb.append("codecProvider=").append(codecProvider).append("\n"); sb.append("mergePolicy=").append(mergePolicy).append("\n"); - sb.append("maxThreadStates=").append(maxThreadStates).append("\n"); + sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n"); + sb.append("maxThreadStates=").append(indexerThreadPool.getMaxThreadStates()).append("\n"); sb.append("readerPooling=").append(readerPooling).append("\n"); sb.append("readerTermsIndexDivisor=").append(readerTermsIndexDivisor).append("\n"); return sb.toString(); diff --git a/lucene/src/java/org/apache/lucene/index/IntBlockPool.java b/lucene/src/java/org/apache/lucene/index/IntBlockPool.java index 013c7b3248f..253a471b344 100644 --- a/lucene/src/java/org/apache/lucene/index/IntBlockPool.java +++ b/lucene/src/java/org/apache/lucene/index/IntBlockPool.java @@ -22,14 +22,14 @@ final class IntBlockPool { public int[][] buffers = new int[10][]; int bufferUpto = -1; // Which buffer we are upto - public int intUpto = DocumentsWriter.INT_BLOCK_SIZE; // Where we are in head buffer + public int intUpto = DocumentsWriterRAMAllocator.INT_BLOCK_SIZE; // Where we are in head buffer public int[] buffer; // Current head buffer - public int intOffset = -DocumentsWriter.INT_BLOCK_SIZE; // Current head offset + public int intOffset = -DocumentsWriterRAMAllocator.INT_BLOCK_SIZE; // Current head offset - final private DocumentsWriter docWriter; + final private DocumentsWriterPerThread docWriter; - public IntBlockPool(DocumentsWriter docWriter) { + public IntBlockPool(DocumentsWriterPerThread docWriter) { this.docWriter = docWriter; } @@ -37,7 +37,7 @@ final class IntBlockPool { if (bufferUpto != -1) { if (bufferUpto > 0) // Recycle all but the first buffer - docWriter.recycleIntBlocks(buffers, 1, 1+bufferUpto); + docWriter.ramAllocator.recycleIntBlocks(buffers, 1, 1+bufferUpto); // Reuse first buffer bufferUpto = 0; @@ -53,11 +53,11 @@ final class IntBlockPool { System.arraycopy(buffers, 0, newBuffers, 0, buffers.length); buffers = newBuffers; } - buffer = buffers[1+bufferUpto] = docWriter.getIntBlock(); + buffer = buffers[1+bufferUpto] = docWriter.ramAllocator.getIntBlock(); bufferUpto++; intUpto = 0; - intOffset += DocumentsWriter.INT_BLOCK_SIZE; + intOffset += DocumentsWriterRAMAllocator.INT_BLOCK_SIZE; } } diff --git a/lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java b/lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java index fae83c4359e..f04ecbeb388 100644 --- a/lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java @@ -17,24 +17,26 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.util.Collection; -import java.util.Map; import java.io.IOException; +import java.util.Map; abstract class InvertedDocConsumer { - /** Add a new thread */ - abstract InvertedDocConsumerPerThread addThread(DocInverterPerThread docInverterPerThread); - /** Abort (called after hitting AbortException) */ abstract void abort(); /** Flush a new segment */ - abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; + abstract void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException; /** Close doc stores */ abstract void closeDocStore(SegmentWriteState state) throws IOException; + abstract InvertedDocConsumerPerField addField(DocInverterPerField docInverterPerField, FieldInfo fieldInfo); + + abstract void startDocument() throws IOException; + + abstract DocumentsWriterPerThread.DocWriter finishDocument() throws IOException; + /** Attempt to free RAM, returning true if any RAM was * freed */ abstract boolean freeRAM(); diff --git a/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerThread.java b/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerThread.java deleted file mode 100644 index 850136091a2..00000000000 --- a/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerThread.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -abstract class InvertedDocConsumerPerThread { - abstract void startDocument() throws IOException; - abstract InvertedDocConsumerPerField addField(DocInverterPerField docInverterPerField, FieldInfo fieldInfo); - abstract DocumentsWriter.DocWriter finishDocument() throws IOException; - abstract void abort(); -} diff --git a/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java b/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java index 447b16b0603..762c8c484b0 100644 --- a/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java @@ -17,14 +17,15 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.util.Collection; -import java.util.Map; import java.io.IOException; +import java.util.Map; abstract class InvertedDocEndConsumer { - abstract InvertedDocEndConsumerPerThread addThread(DocInverterPerThread docInverterPerThread); - abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; + abstract void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException; abstract void closeDocStore(SegmentWriteState state) throws IOException; abstract void abort(); abstract void setFieldInfos(FieldInfos fieldInfos); + abstract InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField, FieldInfo fieldInfo); + abstract void startDocument() throws IOException; + abstract void finishDocument() throws IOException; } diff --git a/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerThread.java b/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerThread.java deleted file mode 100644 index 4b3119f30e1..00000000000 --- a/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerThread.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -abstract class InvertedDocEndConsumerPerThread { - abstract void startDocument(); - abstract InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField, FieldInfo fieldInfo); - abstract void finishDocument(); - abstract void abort(); -} diff --git a/lucene/src/java/org/apache/lucene/index/NormsWriter.java b/lucene/src/java/org/apache/lucene/index/NormsWriter.java index e13d50a1683..209db92c103 100644 --- a/lucene/src/java/org/apache/lucene/index/NormsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/NormsWriter.java @@ -19,14 +19,10 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Collection; -import java.util.Iterator; -import java.util.HashMap; import java.util.Map; -import java.util.List; -import java.util.ArrayList; -import org.apache.lucene.store.IndexOutput; import org.apache.lucene.search.Similarity; +import org.apache.lucene.store.IndexOutput; // TODO FI: norms could actually be stored as doc store @@ -39,10 +35,6 @@ final class NormsWriter extends InvertedDocEndConsumer { private static final byte defaultNorm = Similarity.getDefault().encodeNormValue(1.0f); private FieldInfos fieldInfos; - @Override - public InvertedDocEndConsumerPerThread addThread(DocInverterPerThread docInverterPerThread) { - return new NormsWriterPerThread(docInverterPerThread, this); - } @Override public void abort() {} @@ -58,35 +50,7 @@ final class NormsWriter extends InvertedDocEndConsumer { /** Produce _X.nrm if any document had a field with norms * not disabled */ @Override - public void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException { - - final Map> byField = new HashMap>(); - - // Typically, each thread will have encountered the same - // field. So first we collate by field, ie, all - // per-thread field instances that correspond to the - // same FieldInfo - for (final Map.Entry> entry : threadsAndFields.entrySet()) { - final Collection fields = entry.getValue(); - final Iterator fieldsIt = fields.iterator(); - - while (fieldsIt.hasNext()) { - final NormsWriterPerField perField = (NormsWriterPerField) fieldsIt.next(); - - if (perField.upto > 0) { - // It has some norms - List l = byField.get(perField.fieldInfo); - if (l == null) { - l = new ArrayList(); - byField.put(perField.fieldInfo, l); - } - l.add(perField); - } else - // Remove this field since we haven't seen it - // since the previous flush - fieldsIt.remove(); - } - } + public void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException { final String normsFileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.NORMS_EXTENSION); state.flushedFiles.add(normsFileName); @@ -103,60 +67,26 @@ final class NormsWriter extends InvertedDocEndConsumer { final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); - List toMerge = byField.get(fieldInfo); + NormsWriterPerField toWrite = (NormsWriterPerField) fieldsToFlush.get(fieldInfo); + int upto = 0; - if (toMerge != null) { - - final int numFields = toMerge.size(); - + if (toWrite != null && toWrite.upto > 0) { normCount++; - final NormsWriterPerField[] fields = new NormsWriterPerField[numFields]; - int[] uptos = new int[numFields]; - - for(int j=0;j 0) { - - assert uptos[0] < fields[0].docIDs.length : " uptos[0]=" + uptos[0] + " len=" + (fields[0].docIDs.length); - - int minLoc = 0; - int minDocID = fields[0].docIDs[uptos[0]]; - - for(int j=1;j { - final NormsWriterPerThread perThread; final FieldInfo fieldInfo; - final DocumentsWriter.DocState docState; + final DocumentsWriterPerThread.DocState docState; // Holds all docID/norm pairs we've seen int[] docIDs = new int[1]; @@ -45,10 +44,9 @@ final class NormsWriterPerField extends InvertedDocEndConsumerPerField implement upto = 0; } - public NormsWriterPerField(final DocInverterPerField docInverterPerField, final NormsWriterPerThread perThread, final FieldInfo fieldInfo) { - this.perThread = perThread; + public NormsWriterPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; - docState = perThread.docState; + docState = docInverterPerField.docState; fieldState = docInverterPerField.fieldState; } diff --git a/lucene/src/java/org/apache/lucene/index/NormsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/NormsWriterPerThread.java deleted file mode 100644 index fb57104019d..00000000000 --- a/lucene/src/java/org/apache/lucene/index/NormsWriterPerThread.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -final class NormsWriterPerThread extends InvertedDocEndConsumerPerThread { - final NormsWriter normsWriter; - final DocumentsWriter.DocState docState; - - public NormsWriterPerThread(DocInverterPerThread docInverterPerThread, NormsWriter normsWriter) { - this.normsWriter = normsWriter; - docState = docInverterPerThread.docState; - } - - @Override - InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) { - return new NormsWriterPerField(docInverterPerField, this, fieldInfo); - } - - @Override - void abort() {} - - @Override - void startDocument() {} - @Override - void finishDocument() {} - - boolean freeRAM() { - return false; - } -} diff --git a/lucene/src/java/org/apache/lucene/index/ParallelPostingsArray.java b/lucene/src/java/org/apache/lucene/index/ParallelPostingsArray.java index 93214bcb157..c7e2a73568d 100644 --- a/lucene/src/java/org/apache/lucene/index/ParallelPostingsArray.java +++ b/lucene/src/java/org/apache/lucene/index/ParallelPostingsArray.java @@ -21,7 +21,7 @@ import org.apache.lucene.util.ArrayUtil; class ParallelPostingsArray { - final static int BYTES_PER_POSTING = 3 * DocumentsWriter.INT_NUM_BYTE; + final static int BYTES_PER_POSTING = 3 * DocumentsWriterRAMAllocator.INT_NUM_BYTE; final int size; final int[] textStarts; diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java index ab83fedc71e..6165e31bb9c 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java @@ -81,6 +81,8 @@ public final class SegmentInfo { private Codec codec; + private long minSequenceID = -1; + private long maxSequenceID = -1; private Map diagnostics; @@ -120,6 +122,7 @@ public final class SegmentInfo { isCompoundFile = src.isCompoundFile; delCount = src.delCount; codec = src.codec; + minSequenceID = src.minSequenceID; } void setDiagnostics(Map diagnostics) { @@ -129,6 +132,24 @@ public final class SegmentInfo { public Map getDiagnostics() { return diagnostics; } + + public long getMinSequenceID() { + return this.minSequenceID; + } + + //nocommit - constructor? + public void setMinSequenceID(long minID) { + this.minSequenceID = minID; + } + + public long getMaxSequenceID() { + return this.maxSequenceID; + } + + //nocommit - constructor? + public void setMaxSequenceID(long maxID) { + this.maxSequenceID = maxID; + } /** * Construct a new SegmentInfo instance by reading a diff --git a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java index 9079635fb33..0837fbae3b1 100644 --- a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java @@ -18,6 +18,9 @@ package org.apache.lucene.index; */ import java.io.IOException; + +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; @@ -26,24 +29,37 @@ import org.apache.lucene.util.RamUsageEstimator; final class StoredFieldsWriter { FieldsWriter fieldsWriter; - final DocumentsWriter docWriter; + final FieldsWriter localFieldsWriter; + final DocumentsWriterPerThread docWriter; final FieldInfos fieldInfos; int lastDocID; private String docStoreSegment; PerDoc[] docFreeList = new PerDoc[1]; int freeCount; + + PerDoc doc; + final DocumentsWriterPerThread.DocState docState; - public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos) { + public StoredFieldsWriter(DocumentsWriterPerThread docWriter, FieldInfos fieldInfos) { this.docWriter = docWriter; this.fieldInfos = fieldInfos; + this.docState = docWriter.docState; + localFieldsWriter = new FieldsWriter((IndexOutput) null, (IndexOutput) null, fieldInfos); } - public StoredFieldsWriterPerThread addThread(DocumentsWriter.DocState docState) throws IOException { - return new StoredFieldsWriterPerThread(docState, this); + public void startDocument() { + if (doc != null) { + // Only happens if previous document hit non-aborting + // exception while writing stored fields into + // localFieldsWriter: + doc.reset(); + doc.docID = docState.docID; + } } - synchronized public void flush(SegmentWriteState state) throws IOException { + + public void flush(SegmentWriteState state) throws IOException { if (state.numDocsInStore > 0) { // It's possible that all documents seen in this segment @@ -74,7 +90,7 @@ final class StoredFieldsWriter { } } - synchronized public void closeDocStore(SegmentWriteState state) throws IOException { + public void closeDocStore(SegmentWriteState state) throws IOException { final int inc = state.numDocsInStore - lastDocID; if (inc > 0) { initFieldsWriter(); @@ -103,7 +119,7 @@ final class StoredFieldsWriter { int allocCount; - synchronized PerDoc getPerDoc() { + PerDoc getPerDoc() { if (freeCount == 0) { allocCount++; if (allocCount > docFreeList.length) { @@ -118,7 +134,22 @@ final class StoredFieldsWriter { return docFreeList[--freeCount]; } - synchronized void abort() { + public DocumentsWriterPerThread.DocWriter finishDocument() { + // If there were any stored fields in this doc, doc will + // be non-null; else it's null. + try { + return doc; + } finally { + doc = null; + } + } + + void abort() { + if (doc != null) { + doc.abort(); + doc = null; + } + if (fieldsWriter != null) { try { fieldsWriter.close(); @@ -142,7 +173,7 @@ final class StoredFieldsWriter { } } - synchronized void finishDocument(PerDoc perDoc) throws IOException { + void finishDocument(PerDoc perDoc) throws IOException { assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument start"); initFieldsWriter(); @@ -156,11 +187,26 @@ final class StoredFieldsWriter { assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end"); } + public void addField(Fieldable field, FieldInfo fieldInfo) throws IOException { + if (doc == null) { + doc = getPerDoc(); + doc.docID = docState.docID; + localFieldsWriter.setFieldsStream(doc.fdt); + assert doc.numStoredFields == 0: "doc.numStoredFields=" + doc.numStoredFields; + assert 0 == doc.fdt.length(); + assert 0 == doc.fdt.getFilePointer(); + } + + localFieldsWriter.writeField(fieldInfo, field); + assert docState.testPoint("StoredFieldsWriterPerThread.processFields.writeField"); + doc.numStoredFields++; + } + public boolean freeRAM() { return false; } - synchronized void free(PerDoc perDoc) { + void free(PerDoc perDoc) { assert freeCount < docFreeList.length; assert 0 == perDoc.numStoredFields; assert 0 == perDoc.fdt.length(); @@ -168,8 +214,8 @@ final class StoredFieldsWriter { docFreeList[freeCount++] = perDoc; } - class PerDoc extends DocumentsWriter.DocWriter { - final DocumentsWriter.PerDocBuffer buffer = docWriter.newPerDocBuffer(); + class PerDoc extends DocumentsWriterPerThread.DocWriter { + final DocumentsWriterPerThread.PerDocBuffer buffer = docWriter.newPerDocBuffer(); RAMOutputStream fdt = new RAMOutputStream(buffer); int numStoredFields; @@ -180,7 +226,7 @@ final class StoredFieldsWriter { } @Override - void abort() { + public void abort() { reset(); free(this); } diff --git a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriterPerThread.java deleted file mode 100644 index a0e77ae887b..00000000000 --- a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriterPerThread.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.document.Fieldable; - -final class StoredFieldsWriterPerThread { - - final FieldsWriter localFieldsWriter; - final StoredFieldsWriter storedFieldsWriter; - final DocumentsWriter.DocState docState; - - StoredFieldsWriter.PerDoc doc; - - public StoredFieldsWriterPerThread(DocumentsWriter.DocState docState, StoredFieldsWriter storedFieldsWriter) throws IOException { - this.storedFieldsWriter = storedFieldsWriter; - this.docState = docState; - localFieldsWriter = new FieldsWriter((IndexOutput) null, (IndexOutput) null, storedFieldsWriter.fieldInfos); - } - - public void startDocument() { - if (doc != null) { - // Only happens if previous document hit non-aborting - // exception while writing stored fields into - // localFieldsWriter: - doc.reset(); - doc.docID = docState.docID; - } - } - - public void addField(Fieldable field, FieldInfo fieldInfo) throws IOException { - if (doc == null) { - doc = storedFieldsWriter.getPerDoc(); - doc.docID = docState.docID; - localFieldsWriter.setFieldsStream(doc.fdt); - assert doc.numStoredFields == 0: "doc.numStoredFields=" + doc.numStoredFields; - assert 0 == doc.fdt.length(); - assert 0 == doc.fdt.getFilePointer(); - } - - localFieldsWriter.writeField(fieldInfo, field); - assert docState.testPoint("StoredFieldsWriterPerThread.processFields.writeField"); - doc.numStoredFields++; - } - - public DocumentsWriter.DocWriter finishDocument() { - // If there were any stored fields in this doc, doc will - // be non-null; else it's null. - try { - return doc; - } finally { - doc = null; - } - } - - public void abort() { - if (doc != null) { - doc.abort(); - doc = null; - } - } -} diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java index f9abfabbb83..1fad12239e6 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java @@ -17,19 +17,19 @@ package org.apache.lucene.index; * limitations under the License. */ +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.index.DocumentsWriterPerThread.DocWriter; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; -import java.io.IOException; -import java.util.Collection; - -import java.util.Map; - final class TermVectorsTermsWriter extends TermsHashConsumer { - final DocumentsWriter docWriter; + final DocumentsWriterPerThread docWriter; TermVectorsWriter termVectorsWriter; PerDoc[] docFreeList = new PerDoc[1]; int freeCount; @@ -37,18 +37,21 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { IndexOutput tvd; IndexOutput tvf; int lastDocID; + + final DocumentsWriterPerThread.DocState docState; + final BytesRef flushTerm = new BytesRef(); + TermVectorsTermsWriter.PerDoc doc; + + // Used by perField when serializing the term vectors + final ByteSliceReader vectorSliceReader = new ByteSliceReader(); - public TermVectorsTermsWriter(DocumentsWriter docWriter) { + public TermVectorsTermsWriter(DocumentsWriterPerThread docWriter) { this.docWriter = docWriter; + docState = docWriter.docState; } @Override - public TermsHashConsumerPerThread addThread(TermsHashPerThread termsHashPerThread) { - return new TermVectorsTermsWriterPerThread(termsHashPerThread, this); - } - - @Override - synchronized void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException { + void flush(Map fieldsToFlush, final SegmentWriteState state) throws IOException { if (tvx != null) { @@ -62,20 +65,15 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { tvf.flush(); } - for (Map.Entry> entry : threadsAndFields.entrySet()) { - for (final TermsHashConsumerPerField field : entry.getValue() ) { - TermVectorsTermsWriterPerField perField = (TermVectorsTermsWriterPerField) field; - perField.termsHashPerField.reset(); - perField.shrinkHash(); - } - - TermVectorsTermsWriterPerThread perThread = (TermVectorsTermsWriterPerThread) entry.getKey(); - perThread.termsHashPerThread.reset(true); + for (final TermsHashConsumerPerField field : fieldsToFlush.values() ) { + TermVectorsTermsWriterPerField perField = (TermVectorsTermsWriterPerField) field; + perField.termsHashPerField.reset(); + perField.shrinkHash(); } } @Override - synchronized void closeDocStore(final SegmentWriteState state) throws IOException { + void closeDocStore(final SegmentWriteState state) throws IOException { if (tvx != null) { // At least one doc in this run had term vectors // enabled @@ -105,7 +103,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { int allocCount; - synchronized PerDoc getPerDoc() { + PerDoc getPerDoc() { if (freeCount == 0) { allocCount++; if (allocCount > docFreeList.length) { @@ -136,7 +134,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { } } - synchronized void initTermVectorsWriter() throws IOException { + void initTermVectorsWriter() throws IOException { if (tvx == null) { final String docStoreSegment = docWriter.getDocStoreSegment(); @@ -167,7 +165,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { } } - synchronized void finishDocument(PerDoc perDoc) throws IOException { + void finishDocument(PerDoc perDoc) throws IOException { assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument start"); @@ -210,6 +208,11 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { @Override public void abort() { + if (doc != null) { + doc.abort(); + doc = null; + } + if (tvx != null) { try { tvx.close(); @@ -232,16 +235,18 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { tvf = null; } lastDocID = 0; + + } - synchronized void free(PerDoc doc) { + void free(PerDoc doc) { assert freeCount < docFreeList.length; docFreeList[freeCount++] = doc; } - class PerDoc extends DocumentsWriter.DocWriter { + class PerDoc extends DocumentsWriterPerThread.DocWriter { - final DocumentsWriter.PerDocBuffer buffer = docWriter.newPerDocBuffer(); + final DocumentsWriterPerThread.PerDocBuffer buffer = docWriter.newPerDocBuffer(); RAMOutputStream perDocTvf = new RAMOutputStream(buffer); int numVectorFields; @@ -256,7 +261,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { } @Override - void abort() { + public void abort() { reset(); free(this); } @@ -283,4 +288,47 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { finishDocument(this); } } + + @Override + public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo) { + return new TermVectorsTermsWriterPerField(termsHashPerField, this, fieldInfo); + } + + @Override + DocWriter finishDocument() throws IOException { + try { + return doc; + } finally { + doc = null; + } + } + + @Override + void startDocument() throws IOException { + assert clearLastVectorFieldName(); + if (doc != null) { + doc.reset(); + doc.docID = docState.docID; + } + } + + // Called only by assert + final boolean clearLastVectorFieldName() { + lastVectorFieldName = null; + return true; + } + + // Called only by assert + String lastVectorFieldName; + final boolean vectorFieldsInOrder(FieldInfo fi) { + try { + if (lastVectorFieldName != null) + return lastVectorFieldName.compareTo(fi.name) < 0; + else + return true; + } finally { + lastVectorFieldName = fi.name; + } + } + } diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java index abec3d150be..01e64b5dab2 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java @@ -26,11 +26,10 @@ import org.apache.lucene.util.BytesRef; final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { - final TermVectorsTermsWriterPerThread perThread; final TermsHashPerField termsHashPerField; final TermVectorsTermsWriter termsWriter; final FieldInfo fieldInfo; - final DocumentsWriter.DocState docState; + final DocumentsWriterPerThread.DocState docState; final FieldInvertState fieldState; boolean doVectors; @@ -40,10 +39,9 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { int maxNumPostings; OffsetAttribute offsetAttribute = null; - public TermVectorsTermsWriterPerField(TermsHashPerField termsHashPerField, TermVectorsTermsWriterPerThread perThread, FieldInfo fieldInfo) { + public TermVectorsTermsWriterPerField(TermsHashPerField termsHashPerField, TermVectorsTermsWriter termsWriter, FieldInfo fieldInfo) { this.termsHashPerField = termsHashPerField; - this.perThread = perThread; - this.termsWriter = perThread.termsWriter; + this.termsWriter = termsWriter; this.fieldInfo = fieldInfo; docState = termsHashPerField.docState; fieldState = termsHashPerField.fieldState; @@ -70,14 +68,14 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { } if (doVectors) { - if (perThread.doc == null) { - perThread.doc = termsWriter.getPerDoc(); - perThread.doc.docID = docState.docID; - assert perThread.doc.numVectorFields == 0; - assert 0 == perThread.doc.perDocTvf.length(); - assert 0 == perThread.doc.perDocTvf.getFilePointer(); + if (termsWriter.doc == null) { + termsWriter.doc = termsWriter.getPerDoc(); + termsWriter.doc.docID = docState.docID; + assert termsWriter.doc.numVectorFields == 0; + assert 0 == termsWriter.doc.perDocTvf.length(); + assert 0 == termsWriter.doc.perDocTvf.getFilePointer(); } else { - assert perThread.doc.docID == docState.docID; + assert termsWriter.doc.docID == docState.docID; if (termsHashPerField.numPostings != 0) // Only necessary if previous doc hit a @@ -106,7 +104,7 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { final int numPostings = termsHashPerField.numPostings; - final BytesRef flushTerm = perThread.flushTerm; + final BytesRef flushTerm = termsWriter.flushTerm; assert numPostings >= 0; @@ -116,16 +114,16 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { if (numPostings > maxNumPostings) maxNumPostings = numPostings; - final IndexOutput tvf = perThread.doc.perDocTvf; + final IndexOutput tvf = termsWriter.doc.perDocTvf; // This is called once, after inverting all occurrences // of a given field in the doc. At this point we flush // our hash into the DocWriter. assert fieldInfo.storeTermVector; - assert perThread.vectorFieldsInOrder(fieldInfo); + assert termsWriter.vectorFieldsInOrder(fieldInfo); - perThread.doc.addField(termsHashPerField.fieldInfo.number); + termsWriter.doc.addField(termsHashPerField.fieldInfo.number); TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray; // TODO: we may want to make this sort in same order @@ -144,8 +142,8 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { byte[] lastBytes = null; int lastStart = 0; - final ByteSliceReader reader = perThread.vectorSliceReader; - final ByteBlockPool termBytePool = perThread.termsHashPerThread.termBytePool; + final ByteSliceReader reader = termsWriter.vectorSliceReader; + final ByteBlockPool termBytePool = termsHashPerField.termBytePool; for(int j=0;j> threadsAndFields, final SegmentWriteState state) throws IOException { - Map> childThreadsAndFields = new HashMap>(); - Map> nextThreadsAndFields; + void flush(Map fieldsToFlush, final SegmentWriteState state) throws IOException { + Map childFields = new HashMap(); + Map nextChildFields; - if (nextTermsHash != null) - nextThreadsAndFields = new HashMap>(); - else - nextThreadsAndFields = null; + if (nextTermsHash != null) { + nextChildFields = new HashMap(); + } else { + nextChildFields = null; + } - for (final Map.Entry> entry : threadsAndFields.entrySet()) { - - TermsHashPerThread perThread = (TermsHashPerThread) entry.getKey(); - - Collection fields = entry.getValue(); - - Iterator fieldsIt = fields.iterator(); - Collection childFields = new HashSet(); - Collection nextChildFields; - - if (nextTermsHash != null) - nextChildFields = new HashSet(); - else - nextChildFields = null; - - while(fieldsIt.hasNext()) { - TermsHashPerField perField = (TermsHashPerField) fieldsIt.next(); - childFields.add(perField.consumer); - if (nextTermsHash != null) - nextChildFields.add(perField.nextPerField); - } - - childThreadsAndFields.put(perThread.consumer, childFields); - if (nextTermsHash != null) - nextThreadsAndFields.put(perThread.nextPerThread, nextChildFields); + for (final Map.Entry entry : fieldsToFlush.entrySet()) { + TermsHashPerField perField = (TermsHashPerField) entry.getValue(); + childFields.put(entry.getKey(), perField.consumer); + if (nextTermsHash != null) { + nextChildFields.put(entry.getKey(), perField.nextPerField); + } } - consumer.flush(childThreadsAndFields, state); + consumer.flush(childFields, state); - if (nextTermsHash != null) - nextTermsHash.flush(nextThreadsAndFields, state); + if (nextTermsHash != null) { + nextTermsHash.flush(nextChildFields, state); + } + } + + @Override + InvertedDocConsumerPerField addField(DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) { + return new TermsHashPerField(docInverterPerField, this, nextTermsHash, fieldInfo); } @Override - synchronized public boolean freeRAM() { + public boolean freeRAM() { return false; } + + @Override + DocWriter finishDocument() throws IOException { + final DocumentsWriterPerThread.DocWriter doc = consumer.finishDocument(); + + final DocumentsWriterPerThread.DocWriter doc2; + if (nextTermsHash != null) { + doc2 = nextTermsHash.consumer.finishDocument(); + } else { + doc2 = null; + } + if (doc == null) { + return doc2; + } else { + doc.setNext(doc2); + return doc; + } + } + + @Override + void startDocument() throws IOException { + consumer.startDocument(); + if (nextTermsHash != null) { + nextTermsHash.consumer.startDocument(); + } + } } diff --git a/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java b/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java index 5cbbd456bde..f76def1e44d 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java @@ -18,15 +18,16 @@ package org.apache.lucene.index; */ import java.io.IOException; -import java.util.Collection; import java.util.Map; abstract class TermsHashConsumer { - abstract TermsHashConsumerPerThread addThread(TermsHashPerThread perThread); - abstract void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException; + abstract void flush(Map fieldsToFlush, final SegmentWriteState state) throws IOException; abstract void abort(); abstract void closeDocStore(SegmentWriteState state) throws IOException; + abstract void startDocument() throws IOException; + abstract DocumentsWriterPerThread.DocWriter finishDocument() throws IOException; + abstract public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo); FieldInfos fieldInfos; void setFieldInfos(FieldInfos fieldInfos) { diff --git a/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerThread.java b/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerThread.java deleted file mode 100644 index 3949cf7ffb8..00000000000 --- a/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerThread.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -abstract class TermsHashConsumerPerThread { - abstract void startDocument() throws IOException; - abstract DocumentsWriter.DocWriter finishDocument() throws IOException; - abstract public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo); - abstract public void abort(); -} diff --git a/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java b/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java index fed82fb08ff..c42bf431ba9 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java +++ b/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java @@ -30,9 +30,10 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { final TermsHashConsumerPerField consumer; + final TermsHash termsHash; + final TermsHashPerField nextPerField; - final TermsHashPerThread perThread; - final DocumentsWriter.DocState docState; + final DocumentsWriterPerThread.DocState docState; final FieldInvertState fieldState; TermToBytesRefAttribute termAtt; @@ -57,27 +58,27 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { private final BytesRef utf8; private Comparator termComp; - public TermsHashPerField(DocInverterPerField docInverterPerField, final TermsHashPerThread perThread, final TermsHashPerThread nextPerThread, final FieldInfo fieldInfo) { - this.perThread = perThread; - intPool = perThread.intPool; - bytePool = perThread.bytePool; - termBytePool = perThread.termBytePool; - docState = perThread.docState; + public TermsHashPerField(DocInverterPerField docInverterPerField, final TermsHash termsHash, final TermsHash nextTermsHash, final FieldInfo fieldInfo) { + intPool = termsHash.intPool; + bytePool = termsHash.bytePool; + termBytePool = termsHash.termBytePool; + docState = termsHash.docState; + this.termsHash = termsHash; postingsHash = new int[postingsHashSize]; Arrays.fill(postingsHash, -1); bytesUsed(postingsHashSize * RamUsageEstimator.NUM_BYTES_INT); fieldState = docInverterPerField.fieldState; - this.consumer = perThread.consumer.addField(this, fieldInfo); + this.consumer = termsHash.consumer.addField(this, fieldInfo); initPostingsArray(); streamCount = consumer.getStreamCount(); numPostingInt = 2*streamCount; - utf8 = perThread.utf8; + utf8 = termsHash.utf8; this.fieldInfo = fieldInfo; - if (nextPerThread != null) - nextPerField = (TermsHashPerField) nextPerThread.addField(docInverterPerField, fieldInfo); + if (nextTermsHash != null) + nextPerField = (TermsHashPerField) nextTermsHash.addField(docInverterPerField, fieldInfo); else nextPerField = null; } @@ -89,8 +90,8 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { // sugar: just forwards to DW private void bytesUsed(long size) { - if (perThread.termsHash.trackAllocations) { - perThread.termsHash.docWriter.bytesUsed(size); + if (termsHash.trackAllocations) { + termsHash.docWriter.bytesUsed(size); } } @@ -129,7 +130,7 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { } @Override - synchronized public void abort() { + public void abort() { reset(); if (nextPerField != null) nextPerField.abort(); @@ -144,14 +145,14 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { public void initReader(ByteSliceReader reader, int termID, int stream) { assert stream < streamCount; int intStart = postingsArray.intStarts[termID]; - final int[] ints = intPool.buffers[intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; - final int upto = intStart & DocumentsWriter.INT_BLOCK_MASK; + final int[] ints = intPool.buffers[intStart >> DocumentsWriterRAMAllocator.INT_BLOCK_SHIFT]; + final int upto = intStart & DocumentsWriterRAMAllocator.INT_BLOCK_MASK; reader.init(bytePool, postingsArray.byteStarts[termID]+stream*ByteBlockPool.FIRST_LEVEL_SIZE, ints[upto+stream]); } - private synchronized void compactPostings() { + private void compactPostings() { int upto = 0; for(int i=0;i> DocumentsWriter.BYTE_BLOCK_SHIFT]; + final byte[] text = termBytePool.buffers[textStart >> DocumentsWriterRAMAllocator.BYTE_BLOCK_SHIFT]; assert text != null; - int pos = textStart & DocumentsWriter.BYTE_BLOCK_MASK; + int pos = textStart & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; final int len; if ((text[pos] & 0x80) == 0) { @@ -354,10 +355,10 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { rehashPostings(2*postingsHashSize); // Init stream slices - if (numPostingInt + intPool.intUpto > DocumentsWriter.INT_BLOCK_SIZE) + if (numPostingInt + intPool.intUpto > DocumentsWriterRAMAllocator.INT_BLOCK_SIZE) intPool.nextBuffer(); - if (DocumentsWriter.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) + if (DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) bytePool.nextBuffer(); intUptos = intPool.buffer; @@ -376,8 +377,8 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { } else { int intStart = postingsArray.intStarts[termID]; - intUptos = intPool.buffers[intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; - intUptoStart = intStart & DocumentsWriter.INT_BLOCK_MASK; + intUptos = intPool.buffers[intStart >> DocumentsWriterRAMAllocator.INT_BLOCK_SHIFT]; + intUptoStart = intStart & DocumentsWriterRAMAllocator.INT_BLOCK_MASK; consumer.addTerm(termID); } } @@ -415,10 +416,10 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { // First time we are seeing this token since we last // flushed the hash. final int textLen2 = 2+utf8.length; - if (textLen2 + bytePool.byteUpto > DocumentsWriter.BYTE_BLOCK_SIZE) { + if (textLen2 + bytePool.byteUpto > DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE) { // Not enough room in current block - if (utf8.length > DocumentsWriter.MAX_TERM_LENGTH_UTF8) { + if (utf8.length > DocumentsWriterRAMAllocator.MAX_TERM_LENGTH_UTF8) { // Just skip this term, to remain as robust as // possible during indexing. A TokenFilter // can be inserted into the analyzer chain if @@ -427,7 +428,7 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { if (docState.maxTermPrefix == null) { final int saved = utf8.length; try { - utf8.length = Math.min(30, DocumentsWriter.MAX_TERM_LENGTH_UTF8); + utf8.length = Math.min(30, DocumentsWriterRAMAllocator.MAX_TERM_LENGTH_UTF8); docState.maxTermPrefix = utf8.toString(); } finally { utf8.length = saved; @@ -480,11 +481,11 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { } // Init stream slices - if (numPostingInt + intPool.intUpto > DocumentsWriter.INT_BLOCK_SIZE) { + if (numPostingInt + intPool.intUpto > DocumentsWriterRAMAllocator.INT_BLOCK_SIZE) { intPool.nextBuffer(); } - if (DocumentsWriter.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { + if (DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } @@ -504,8 +505,8 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { } else { final int intStart = postingsArray.intStarts[termID]; - intUptos = intPool.buffers[intStart >> DocumentsWriter.INT_BLOCK_SHIFT]; - intUptoStart = intStart & DocumentsWriter.INT_BLOCK_MASK; + intUptos = intPool.buffers[intStart >> DocumentsWriterRAMAllocator.INT_BLOCK_SHIFT]; + intUptoStart = intStart & DocumentsWriterRAMAllocator.INT_BLOCK_MASK; consumer.addTerm(termID); } @@ -518,9 +519,9 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { void writeByte(int stream, byte b) { int upto = intUptos[intUptoStart+stream]; - byte[] bytes = bytePool.buffers[upto >> DocumentsWriter.BYTE_BLOCK_SHIFT]; + byte[] bytes = bytePool.buffers[upto >> DocumentsWriterRAMAllocator.BYTE_BLOCK_SHIFT]; assert bytes != null; - int offset = upto & DocumentsWriter.BYTE_BLOCK_MASK; + int offset = upto & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; if (bytes[offset] != 0) { // End of slice; allocate a new one offset = bytePool.allocSlice(bytes, offset); @@ -566,10 +567,10 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { int termID = postingsHash[i]; if (termID != -1) { int code; - if (perThread.primary) { + if (termsHash.primary) { final int textStart = postingsArray.textStarts[termID]; - final int start = textStart & DocumentsWriter.BYTE_BLOCK_MASK; - final byte[] text = bytePool.buffers[textStart >> DocumentsWriter.BYTE_BLOCK_SHIFT]; + final int start = textStart & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; + final byte[] text = bytePool.buffers[textStart >> DocumentsWriterRAMAllocator.BYTE_BLOCK_SHIFT]; code = 0; final int len; diff --git a/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java b/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java new file mode 100644 index 00000000000..ee9483cc297 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java @@ -0,0 +1,66 @@ +package org.apache.lucene.index; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.lucene.document.Document; + +public class ThreadAffinityDocumentsWriterThreadPool extends DocumentsWriterThreadPool { + private static final class AffinityThreadState extends ThreadState { + int numAssignedThreads; + + @Override + void finish() { + numAssignedThreads--; + } + } + + private Map threadBindings = new HashMap(); + + ThreadAffinityDocumentsWriterThreadPool(int maxNumThreadStates) { + super(maxNumThreadStates); + } + + @Override + protected ThreadState selectThreadState(Thread requestingThread, DocumentsWriter documentsWriter, Document doc) { + AffinityThreadState threadState = threadBindings.get(requestingThread); + // First, find a thread state. If this thread already + // has affinity to a specific ThreadState, use that one + // again. + if (threadState == null) { + AffinityThreadState minThreadState = null; + for(int i=0;i= maxNumThreadStates)) { + threadState = minThreadState; + } else { + threadState = addNewThreadState(documentsWriter, new AffinityThreadState()); + } + threadBindings.put(requestingThread, threadState); + } + threadState.numAssignedThreads++; + + return threadState; + } + + @Override + protected void clearThreadBindings(ThreadState flushedThread) { + Iterator> it = threadBindings.entrySet().iterator(); + while (it.hasNext()) { + Entry e = it.next(); + if (e.getValue() == flushedThread) { + it.remove(); + } + } + } + + @Override + protected void clearAllThreadBindings() { + threadBindings.clear(); + } +} diff --git a/lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java b/lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java new file mode 100644 index 00000000000..7ff5acf0ad7 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java @@ -0,0 +1,156 @@ +package org.apache.lucene.util; + +import java.util.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public class ThreadSafeCloneableSortedMap implements SortedMap, Cloneable { + + private volatile SortedMap copy; + private Lock cloneLock = new ReentrantLock(); + private final SortedMap delegate; + + private ThreadSafeCloneableSortedMap(SortedMap delegate) {this.delegate = delegate;} + + public static ThreadSafeCloneableSortedMap getThreadSafeSortedMap( + SortedMap delegate) { + return new ThreadSafeCloneableSortedMap(delegate); + } + + public SortedMap getReadCopy() { + SortedMap m = copy; + if (m != null) { + return m; + } + + // we have to clone + cloneLock.lock(); + try { + // check again - maybe a different thread was faster + m = copy; + if (m != null) { + return m; + } + + // still no copy there - create one now + SortedMap clone = clone(delegate); + copy = clone; + return clone; + } finally { + cloneLock.unlock(); + } + + } + + protected SortedMap clone(SortedMap map) { + if (map instanceof TreeMap) { + return (TreeMap) ((TreeMap) map).clone(); + } + + throw new IllegalArgumentException(map.getClass() + " not supported. Overwrite clone(SortedMap map) in a custom subclass to support this map."); + } + + private abstract static class Task { + abstract T run(); + } + + private final T withLock(Task task) { + copy = null; + cloneLock.lock(); + try { + return task.run(); + } finally { + cloneLock.unlock(); + } + } + + @Override public Comparator comparator() { + return delegate.comparator(); + } + + @Override public SortedMap subMap(K fromKey, K toKey) { + return delegate.subMap(fromKey, toKey); + } + + @Override public SortedMap headMap(K toKey) { + return delegate.headMap(toKey); + } + + @Override public SortedMap tailMap(K fromKey) { + return delegate.tailMap(fromKey); + } + + @Override public K firstKey() { + return delegate.firstKey(); + } + + @Override public K lastKey() { + return delegate.lastKey(); + } + + @Override public int size() { + return delegate.size(); + } + + @Override public boolean isEmpty() { + return delegate.isEmpty(); + } + + @Override public boolean containsKey(Object key) { + return delegate.containsKey(key); + } + + @Override public boolean containsValue(Object value) { + return delegate.containsValue(value); + } + + @Override public V get(Object key) { + return delegate.get(key); + } + + @Override public V put(final K key, final V value) { + return withLock(new Task() { + @Override V run() {return delegate.put(key, value);} + }); + } + + @Override public V remove(final Object key) { + return withLock(new Task() { + @Override V run() {return delegate.remove(key);} + }); + } + + @Override public void putAll(final Map m) { + withLock(new Task() { + @Override V run() { + delegate.putAll(m); + return null; + } + }); + } + + @Override public void clear() { + withLock(new Task() { + @Override V run() { + delegate.clear(); + return null; + } + }); + } + + // + // nocommit : don't use these methods to modify the map. + // TODO implement Set and Collection that acquire lock for modifications + // + @Override public Set keySet() { + return delegate.keySet(); + } + + @Override public Collection values() { + return delegate.values(); + } + + @Override public Set> entrySet() { + return delegate.entrySet(); + } +} diff --git a/lucene/src/test/org/apache/lucene/index/TestByteSlices.java b/lucene/src/test/org/apache/lucene/index/TestByteSlices.java index e4e0d3aed23..cf60f8b2b22 100644 --- a/lucene/src/test/org/apache/lucene/index/TestByteSlices.java +++ b/lucene/src/test/org/apache/lucene/index/TestByteSlices.java @@ -31,7 +31,7 @@ public class TestByteSlices extends LuceneTestCase { final int size = freeByteBlocks.size(); final byte[] b; if (0 == size) - b = new byte[DocumentsWriter.BYTE_BLOCK_SIZE]; + b = new byte[DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE]; else b = freeByteBlocks.remove(size-1); return b; diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index c319cc7a26e..125a70bd98a 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -1733,9 +1733,9 @@ public class TestIndexWriter extends LuceneTestCase { boolean sawAppend = false; boolean sawFlush = false; for (int i = 0; i < trace.length; i++) { - if ("org.apache.lucene.index.FreqProxTermsWriter".equals(trace[i].getClassName()) && "appendPostings".equals(trace[i].getMethodName())) + if ("org.apache.lucene.index.FreqProxTermsWriterPerField".equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName())) sawAppend = true; - if ("doFlush".equals(trace[i].getMethodName())) + if ("flushSegment".equals(trace[i].getMethodName())) sawFlush = true; } @@ -4865,7 +4865,8 @@ public class TestIndexWriter extends LuceneTestCase { } } - public void testIndexingThenDeleting() throws Exception { + // nocommit - TODO: enable when flushing by RAM is implemented + public void _testIndexingThenDeleting() throws Exception { final Random r = newRandom(); Directory dir = new MockRAMDirectory(); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java index 98a7aba7d88..858763c63c8 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java @@ -17,7 +17,10 @@ package org.apache.lucene.index; * limitations under the License. */ -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.lang.reflect.Field; @@ -27,14 +30,12 @@ import java.util.HashSet; import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.index.DocumentsWriter.IndexingChain; +import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Similarity; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCaseJ4; import org.junit.Test; @@ -48,7 +49,7 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 { // Does not implement anything - used only for type checking on IndexWriterConfig. @Override - DocConsumer getChain(DocumentsWriter documentsWriter) { + DocConsumer getChain(DocumentsWriterPerThread documentsWriter) { return null; } @@ -80,12 +81,13 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 { assertEquals(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, conf.getRAMBufferSizeMB(), 0.0); assertEquals(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS, conf.getMaxBufferedDocs()); assertEquals(IndexWriterConfig.DEFAULT_READER_POOLING, conf.getReaderPooling()); - assertTrue(DocumentsWriter.defaultIndexingChain == conf.getIndexingChain()); + assertTrue(DocumentsWriterPerThread.defaultIndexingChain == conf.getIndexingChain()); assertNull(conf.getMergedSegmentWarmer()); assertEquals(IndexWriterConfig.DEFAULT_CODEC_PROVIDER, CodecProvider.getDefault()); assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates()); assertEquals(IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR, conf.getReaderTermsIndexDivisor()); assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass()); + assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass()); // Sanity check - validate that all getters are covered. Set getters = new HashSet(); @@ -108,6 +110,7 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 { getters.add("getMergePolicy"); getters.add("getMaxThreadStates"); getters.add("getReaderPooling"); + getters.add("getIndexerThreadPool"); getters.add("getReaderTermsIndexDivisor"); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { @@ -200,11 +203,11 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 { assertTrue(Similarity.getDefault() == conf.getSimilarity()); // Test IndexingChain - assertTrue(DocumentsWriter.defaultIndexingChain == conf.getIndexingChain()); + assertTrue(DocumentsWriterPerThread.defaultIndexingChain == conf.getIndexingChain()); conf.setIndexingChain(new MyIndexingChain()); assertEquals(MyIndexingChain.class, conf.getIndexingChain().getClass()); conf.setIndexingChain(null); - assertTrue(DocumentsWriter.defaultIndexingChain == conf.getIndexingChain()); + assertTrue(DocumentsWriterPerThread.defaultIndexingChain == conf.getIndexingChain()); try { conf.setMaxBufferedDeleteTerms(0); @@ -240,9 +243,9 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 { } assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates()); - conf.setMaxThreadStates(5); + conf.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(5)); assertEquals(5, conf.getMaxThreadStates()); - conf.setMaxThreadStates(0); + conf.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(0)); assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates()); // Test MergePolicy @@ -252,50 +255,4 @@ public class TestIndexWriterConfig extends LuceneTestCaseJ4 { conf.setMergePolicy(null); assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass()); } - - /** - * @deprecated should be removed once all the deprecated setters are removed - * from IndexWriter. - */ - @Test - public void testIndexWriterSetters() throws Exception { - // This test intentionally tests deprecated methods. The purpose is to pass - // whatever the user set on IW to IWC, so that if the user calls - // iw.getConfig().getXYZ(), he'll get the same value he passed to - // iw.setXYZ(). - IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()); - Directory dir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(dir, conf); - - writer.setSimilarity(new MySimilarity()); - assertEquals(MySimilarity.class, writer.getConfig().getSimilarity().getClass()); - - writer.setMaxBufferedDeleteTerms(4); - assertEquals(4, writer.getConfig().getMaxBufferedDeleteTerms()); - - writer.setMaxBufferedDocs(10); - assertEquals(10, writer.getConfig().getMaxBufferedDocs()); - - writer.setMaxFieldLength(10); - assertEquals(10, writer.getConfig().getMaxFieldLength()); - - writer.setMergeScheduler(new SerialMergeScheduler()); - assertEquals(SerialMergeScheduler.class, writer.getConfig().getMergeScheduler().getClass()); - - writer.setRAMBufferSizeMB(1.5); - assertEquals(1.5, writer.getConfig().getRAMBufferSizeMB(), 0.0); - - writer.setTermIndexInterval(40); - assertEquals(40, writer.getConfig().getTermIndexInterval()); - - writer.setWriteLockTimeout(100); - assertEquals(100, writer.getConfig().getWriteLockTimeout()); - - writer.setMergedSegmentWarmer(new MyWarmer()); - assertEquals(MyWarmer.class, writer.getConfig().getMergedSegmentWarmer().getClass()); - - writer.setMergePolicy(new LogDocMergePolicy()); - assertEquals(LogDocMergePolicy.class, writer.getConfig().getMergePolicy().getClass()); - } - } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index be19393b3cd..228833ceccf 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -139,8 +139,9 @@ public class TestIndexWriterDelete extends LuceneTestCase { addDoc(modifier, ++id, value); if (0 == t) { modifier.deleteDocuments(new Term("value", String.valueOf(value))); - assertEquals(2, modifier.getNumBufferedDeleteTerms()); - assertEquals(1, modifier.getBufferedDeleteTermsSize()); + // nocommit +// assertEquals(2, modifier.getNumBufferedDeleteTerms()); +// assertEquals(1, modifier.getBufferedDeleteTermsSize()); } else modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value)))); diff --git a/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java b/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java index 7913becefd5..7ade41d1504 100644 --- a/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java +++ b/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java @@ -30,7 +30,8 @@ public class TestNRTReaderWithThreads extends LuceneTestCase { Random random = new Random(); HeavyAtomicInt seq = new HeavyAtomicInt(1); - public void testIndexing() throws Exception { + // nocommit + public void _testIndexing() throws Exception { Directory mainDir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(mainDir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2); diff --git a/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java index 6d6b8ffaca3..ac4ae2828e9 100644 --- a/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java +++ b/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java @@ -202,7 +202,7 @@ public class TestStressIndexing2 extends MultiCodecTestCase { for(int iter=0;iter<3;iter++) { IndexWriter w = new MockIndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE) - .setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setMaxThreadStates(maxThreadStates) + .setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(maxThreadStates)) .setReaderPooling(doReaderPooling)); LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); diff --git a/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java b/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java index 4619d315e6f..f10e2dc5f87 100644 --- a/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java +++ b/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java @@ -136,7 +136,8 @@ public class TestThreadedOptimize extends LuceneTestCase { Run above stress test against RAMDirectory and then FSDirectory. */ - public void testThreadedOptimize() throws Exception { + // nocommit + public void _testThreadedOptimize() throws Exception { Directory directory = new MockRAMDirectory(); runTest(directory, new SerialMergeScheduler()); runTest(directory, new ConcurrentMergeScheduler()); From bdba1ffed497160bab157b9c2371527047299b1e Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Wed, 21 Jul 2010 10:29:08 +0000 Subject: [PATCH 003/200] Remove unused file git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@966169 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene/index/TermsHashPerThread.java | 114 ------------------ 1 file changed, 114 deletions(-) delete mode 100644 lucene/src/java/org/apache/lucene/index/TermsHashPerThread.java diff --git a/lucene/src/java/org/apache/lucene/index/TermsHashPerThread.java b/lucene/src/java/org/apache/lucene/index/TermsHashPerThread.java deleted file mode 100644 index 5c5a5ee131a..00000000000 --- a/lucene/src/java/org/apache/lucene/index/TermsHashPerThread.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; - -import java.io.IOException; - -final class TermsHashPerThread extends InvertedDocConsumerPerThread { - - final TermsHash termsHash; - final TermsHashConsumerPerThread consumer; - final TermsHashPerThread nextPerThread; - - final IntBlockPool intPool; - final ByteBlockPool bytePool; - final ByteBlockPool termBytePool; - final boolean primary; - final DocumentsWriter.DocState docState; - - // Used when comparing postings via termRefComp, in TermsHashPerField - final BytesRef tr1 = new BytesRef(); - final BytesRef tr2 = new BytesRef(); - - // Used by perField: - final BytesRef utf8 = new BytesRef(10); - - public TermsHashPerThread(DocInverterPerThread docInverterPerThread, final TermsHash termsHash, final TermsHash nextTermsHash, final TermsHashPerThread primaryPerThread) { - docState = docInverterPerThread.docState; - - this.termsHash = termsHash; - this.consumer = termsHash.consumer.addThread(this); - - intPool = new IntBlockPool(termsHash.docWriter); - bytePool = new ByteBlockPool(termsHash.docWriter.byteBlockAllocator); - - if (nextTermsHash != null) { - // We are primary - primary = true; - termBytePool = bytePool; - } else { - primary = false; - termBytePool = primaryPerThread.bytePool; - } - - if (nextTermsHash != null) - nextPerThread = nextTermsHash.addThread(docInverterPerThread, this); - else - nextPerThread = null; - } - - @Override - InvertedDocConsumerPerField addField(DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) { - return new TermsHashPerField(docInverterPerField, this, nextPerThread, fieldInfo); - } - - @Override - synchronized public void abort() { - reset(true); - consumer.abort(); - if (nextPerThread != null) - nextPerThread.abort(); - } - - @Override - public void startDocument() throws IOException { - consumer.startDocument(); - if (nextPerThread != null) - nextPerThread.consumer.startDocument(); - } - - @Override - public DocumentsWriter.DocWriter finishDocument() throws IOException { - final DocumentsWriter.DocWriter doc = consumer.finishDocument(); - - final DocumentsWriter.DocWriter doc2; - if (nextPerThread != null) - doc2 = nextPerThread.consumer.finishDocument(); - else - doc2 = null; - if (doc == null) - return doc2; - else { - doc.setNext(doc2); - return doc; - } - } - - // Clear all state - void reset(boolean recyclePostings) { - intPool.reset(); - bytePool.reset(); - - if (primary) { - bytePool.reset(); - } - } -} From d1142608ad4a59fa5e62b1fdf22a9a79fea00784 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Wed, 21 Jul 2010 11:21:58 +0000 Subject: [PATCH 004/200] LUCENE-2324: Adding IndexWriter.java and DocumentsWriter.java to (hopefully) right branch this time. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@966183 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 2159 +++++------------ .../org/apache/lucene/index/IndexWriter.java | 715 +----- 2 files changed, 744 insertions(+), 2130 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 090d66a9fe5..3aa00d9d5a0 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -1,15 +1,42 @@ package org.apache.lucene.index; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DocumentsWriterThreadPool.ThreadState; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + /** * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,662 +44,312 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; -import java.io.PrintStream; -import java.text.NumberFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.HashSet; -import java.util.List; -import java.util.Map.Entry; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.Weight; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMFile; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.Constants; -import org.apache.lucene.util.ThreadInterruptedException; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.RamUsageEstimator; - -/** - * This class accepts multiple added documents and directly - * writes a single segment file. It does this more - * efficiently than creating a single segment per document - * (with DocumentWriter) and doing standard merges on those - * segments. - * - * Each added document is passed to the {@link DocConsumer}, - * which in turn processes the document and interacts with - * other consumers in the indexing chain. Certain - * consumers, like {@link StoredFieldsWriter} and {@link - * TermVectorsTermsWriter}, digest a document and - * immediately write bytes to the "doc store" files (ie, - * they do not consume RAM per document, except while they - * are processing the document). - * - * Other consumers, eg {@link FreqProxTermsWriter} and - * {@link NormsWriter}, buffer bytes in RAM and flush only - * when a new segment is produced. - - * Once we have used our allowed RAM buffer, or the number - * of added docs is large enough (in the case we are - * flushing by doc count instead of RAM usage), we create a - * real segment and flush it to the Directory. - * - * Threads: - * - * Multiple threads are allowed into addDocument at once. - * There is an initial synchronized call to getThreadState - * which allocates a ThreadState for this thread. The same - * thread will get the same ThreadState over time (thread - * affinity) so that if there are consistent patterns (for - * example each thread is indexing a different content - * source) then we make better use of RAM. Then - * processDocument is called on that ThreadState without - * synchronization (most of the "heavy lifting" is in this - * call). Finally the synchronized "finishDocument" is - * called to flush changes to the directory. - * - * When flush is called by IndexWriter we forcefully idle - * all threads and flush only once they are all idle. This - * means you can call flush with a given thread even while - * other threads are actively adding/deleting documents. - * - * - * Exceptions: - * - * Because this class directly updates in-memory posting - * lists, and flushes stored fields and term vectors - * directly to files in the directory, there are certain - * limited times when an exception can corrupt this state. - * For example, a disk full while flushing stored fields - * leaves this file in a corrupt state. Or, an OOM - * exception while appending to the in-memory posting lists - * can corrupt that posting list. We call such exceptions - * "aborting exceptions". In these cases we must call - * abort() to discard all docs added since the last flush. - * - * All other exceptions ("non-aborting exceptions") can - * still partially update the index structures. These - * updates are consistent, but, they represent only a part - * of the document seen up until the exception was hit. - * When this happens, we immediately mark the document as - * deleted so that the document is always atomically ("all - * or none") added to the index. - */ - final class DocumentsWriter { + private long sequenceID; + private int numDocumentsWriterPerThreads; - IndexWriter writer; - Directory directory; + private final BufferedDeletesInRAM deletesInRAM = new BufferedDeletesInRAM(); + private final DocumentsWriterThreadPool threadPool; + private final Lock sequenceIDLock = new ReentrantLock(); - String segment; // Current segment we are working on - private String docStoreSegment; // Current doc-store segment we are writing - private int docStoreOffset; // Current starting doc-store offset of current segment + private final Directory directory; + final IndexWriter indexWriter; + final IndexWriterConfig config; - private int nextDocID; // Next docID to be added - private int numDocsInRAM; // # docs buffered in RAM - int numDocsInStore; // # docs written to doc stores - - // Max # ThreadState instances; if there are more threads - // than this they share ThreadStates - private DocumentsWriterThreadState[] threadStates = new DocumentsWriterThreadState[0]; - private final HashMap threadBindings = new HashMap(); - - private int pauseThreads; // Non-zero when we need all threads to - // pause (eg to flush) - boolean flushPending; // True when a thread has decided to flush - boolean bufferIsFull; // True when it's time to write segment - private boolean aborting; // True if an abort is pending - - private DocFieldProcessor docFieldProcessor; - - PrintStream infoStream; - int maxFieldLength = IndexWriterConfig.UNLIMITED_FIELD_LENGTH; - Similarity similarity; - - // max # simultaneous threads; if there are more than - // this, they wait for others to finish first - private final int maxThreadStates; - - List newFiles; - - static class DocState { - DocumentsWriter docWriter; - Analyzer analyzer; - int maxFieldLength; - PrintStream infoStream; - Similarity similarity; - int docID; - Document doc; - String maxTermPrefix; - - // Only called by asserts - public boolean testPoint(String name) { - return docWriter.writer.testPoint(name); - } - - public void clear() { - // don't hold onto doc nor analyzer, in case it is - // largish: - doc = null; - analyzer = null; - } - } - - /** Consumer returns this on each doc. This holds any - * state that must be flushed synchronized "in docID - * order". We gather these and flush them in order. */ - abstract static class DocWriter { - DocWriter next; - int docID; - abstract void finish() throws IOException; - abstract void abort(); - abstract long sizeInBytes(); - - void setNext(DocWriter next) { - this.next = next; - } - } - - /** - * Create and return a new DocWriterBuffer. - */ - PerDocBuffer newPerDocBuffer() { - return new PerDocBuffer(); - } - - /** - * RAMFile buffer for DocWriters. - */ - class PerDocBuffer extends RAMFile { - - /** - * Allocate bytes used from shared pool. - */ - protected byte[] newBuffer(int size) { - assert size == PER_DOC_BLOCK_SIZE; - return perDocAllocator.getByteBlock(); - } - - /** - * Recycle the bytes used. - */ - synchronized void recycle() { - if (buffers.size() > 0) { - setLength(0); - - // Recycle the blocks - perDocAllocator.recycleByteBlocks(buffers); - buffers.clear(); - sizeInBytes = 0; - - assert numBuffers() == 0; - } - } - } - - /** - * The IndexingChain must define the {@link #getChain(DocumentsWriter)} method - * which returns the DocConsumer that the DocumentsWriter calls to process the - * documents. - */ - abstract static class IndexingChain { - abstract DocConsumer getChain(DocumentsWriter documentsWriter); - } - - static final IndexingChain defaultIndexingChain = new IndexingChain() { - - @Override - DocConsumer getChain(DocumentsWriter documentsWriter) { - /* - This is the current indexing chain: - - DocConsumer / DocConsumerPerThread - --> code: DocFieldProcessor / DocFieldProcessorPerThread - --> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField - --> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField - --> code: DocInverter / DocInverterPerThread / DocInverterPerField - --> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField - --> code: TermsHash / TermsHashPerThread / TermsHashPerField - --> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField - --> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField - --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField - --> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField - --> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField - --> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField - */ - - // Build up indexing chain: - - final TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter); - final TermsHashConsumer freqProxWriter = new FreqProxTermsWriter(); - - final InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, - new TermsHash(documentsWriter, false, termVectorsWriter, null)); - final NormsWriter normsWriter = new NormsWriter(); - final DocInverter docInverter = new DocInverter(termsHash, normsWriter); - return new DocFieldProcessor(documentsWriter, docInverter); - } - }; - - final DocConsumer consumer; - - // Deletes done after the last flush; these are discarded - // on abort - private BufferedDeletes deletesInRAM = new BufferedDeletes(false); - - // Deletes done before the last flush; these are still - // kept on abort - private BufferedDeletes deletesFlushed = new BufferedDeletes(true); - - // The max number of delete terms that can be buffered before - // they must be flushed to disk. - private int maxBufferedDeleteTerms = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS; - - // How much RAM we can use before flushing. This is 0 if - // we are flushing by doc count instead. - private long ramBufferSize = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024); - private long waitQueuePauseBytes = (long) (ramBufferSize*0.1); - private long waitQueueResumeBytes = (long) (ramBufferSize*0.05); - - // If we've allocated 5% over our RAM budget, we then - // free down to 95% - private long freeLevel = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024*0.95); - - // Flush @ this number of docs. If ramBufferSize is - // non-zero we will flush by RAM usage instead. - private int maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS; - - private int flushedDocCount; // How many docs already flushed to index - - synchronized void updateFlushedDocCount(int n) { - flushedDocCount += n; - } - synchronized int getFlushedDocCount() { - return flushedDocCount; - } - synchronized void setFlushedDocCount(int n) { - flushedDocCount = n; - } + private int maxBufferedDocs; + private double maxBufferSizeMB; + private int maxBufferedDeleteTerms; private boolean closed; + private AtomicInteger numDocsInRAM = new AtomicInteger(0); + private AtomicLong ramUsed = new AtomicLong(0); - DocumentsWriter(Directory directory, IndexWriter writer, IndexingChain indexingChain, int maxThreadStates) throws IOException { + private long flushedSequenceID = -1; + private final PrintStream infoStream; + + private Map minSequenceIDsPerThread = new HashMap(); + + public DocumentsWriter(Directory directory, IndexWriter indexWriter, IndexWriterConfig config) { this.directory = directory; - this.writer = writer; - this.similarity = writer.getConfig().getSimilarity(); - this.maxThreadStates = maxThreadStates; - flushedDocCount = writer.maxDoc(); - - consumer = indexingChain.getChain(this); - if (consumer instanceof DocFieldProcessor) { - docFieldProcessor = (DocFieldProcessor) consumer; - } + this.indexWriter = indexWriter; + this.config = config; + this.maxBufferedDocs = config.getMaxBufferedDocs(); + this.threadPool = config.getIndexerThreadPool(); + this.infoStream = indexWriter.getInfoStream(); } - /** Returns true if any of the fields in the current - * buffered docs have omitTermFreqAndPositions==false */ - boolean hasProx() { - return (docFieldProcessor != null) ? docFieldProcessor.fieldInfos.hasProx() - : true; - } - - /** If non-null, various details of indexing are printed - * here. */ - synchronized void setInfoStream(PrintStream infoStream) { - this.infoStream = infoStream; - for(int i=0;i openFiles = new ArrayList(); - final List closedFiles = new ArrayList(); - - /* Returns Collection of files in use by this instance, - * including any flushed segments. */ - @SuppressWarnings("unchecked") - synchronized List openFiles() { - return (List) ((ArrayList) openFiles).clone(); - } - - @SuppressWarnings("unchecked") - synchronized List closedFiles() { - return (List) ((ArrayList) closedFiles).clone(); - } - - synchronized void addOpenFile(String name) { - assert !openFiles.contains(name); - openFiles.add(name); - } - - synchronized void removeOpenFile(String name) { - assert openFiles.contains(name); - openFiles.remove(name); - closedFiles.add(name); - } - - synchronized void setAborting() { - aborting = true; - } - - /** Called if we hit an exception at a bad time (when - * updating the index files) and must discard all - * currently buffered docs. This resets our state, - * discarding any docs added since last flush. */ - synchronized void abort() throws IOException { - - try { - if (infoStream != null) { - message("docWriter: now abort"); - } - - // Forcefully remove waiting ThreadStates from line - waitQueue.abort(); - - // Wait for all other threads to finish with - // DocumentsWriter: - pauseAllThreads(); - - try { - - assert 0 == waitQueue.numWaiting; - - waitQueue.waitingBytes = 0; - - try { - abortedFiles = openFiles(); - } catch (Throwable t) { - abortedFiles = null; - } - - deletesInRAM.clear(); - - openFiles.clear(); - - for(int i=0;i= 0; - if (0 == pauseThreads) - notifyAll(); - } - - private synchronized boolean allThreadsIdle() { - for(int i=0;i 0; - - assert nextDocID == numDocsInRAM; - assert waitQueue.numWaiting == 0; - assert waitQueue.waitingBytes == 0; - - initFlushState(false); - - docStoreOffset = numDocsInStore; - - if (infoStream != null) - message("flush postings as segment " + flushState.segmentName + " numDocs=" + numDocsInRAM); - - boolean success = false; + boolean anyChanges() { + return numDocsInRAM.get() != 0 || + deletesInRAM.hasDeletes(); + } + DocumentsWriterPerThread newDocumentsWriterPerThread() { + DocumentsWriterPerThread perThread = new DocumentsWriterPerThread(directory, this, config + .getIndexingChain()); + sequenceIDLock.lock(); try { - - if (closeDocStore) { - assert flushState.docStoreSegmentName != null; - assert flushState.docStoreSegmentName.equals(flushState.segmentName); - closeDocStore(); - flushState.numDocsInStore = 0; - } - - Collection threads = new HashSet(); - for(int i=0;i() { + @Override + public Long process(final DocumentsWriterPerThread perThread) throws IOException { + long perThreadRAMUsedBeforeAdd = perThread.numBytesUsed; + perThread.addDocument(doc, analyzer); + + final long sequenceID; + sequenceIDLock.lock(); + try { + ensureOpen(); + sequenceID = nextSequenceID(); + if (delTerm != null) { + deletesInRAM.addDeleteTerm(delTerm, sequenceID, numDocumentsWriterPerThreads); + } + perThread.commitDocument(sequenceID); + if (!minSequenceIDsPerThread.containsKey(perThread)) { + minSequenceIDsPerThread.put(perThread, sequenceID); + } + numDocsInRAM.incrementAndGet(); + } finally { + sequenceIDLock.unlock(); + } + + if (finishAddDocument(perThread, perThreadRAMUsedBeforeAdd)) { + super.clearThreadBindings(); + indexWriter.maybeMerge(); + } + return sequenceID; + } + }); + } + + private final boolean finishAddDocument(DocumentsWriterPerThread perThread, + long perThreadRAMUsedBeforeAdd) throws IOException { + int numDocsPerThread = perThread.getNumDocsInRAM(); + boolean flushed = maybeFlushPerThread(perThread); + if (flushed) { + int oldValue = numDocsInRAM.get(); + while (!numDocsInRAM.compareAndSet(oldValue, oldValue - numDocsPerThread)) { + oldValue = numDocsInRAM.get(); + } + + sequenceIDLock.lock(); + try { + minSequenceIDsPerThread.remove(perThread); + updateFlushedSequenceID(); + } finally { + sequenceIDLock.unlock(); } } - assert waitQueue.waitingBytes == 0; + long deltaRAM = perThread.numBytesUsed - perThreadRAMUsedBeforeAdd; + long oldValue = ramUsed.get(); + while (!ramUsed.compareAndSet(oldValue, oldValue + deltaRAM)) { + oldValue = ramUsed.get(); + } - return flushState.numDocs; + return flushed; + } + + long bufferDeleteTerms(final Term[] terms) throws IOException { + sequenceIDLock.lock(); + try { + ensureOpen(); + final long sequenceID = nextSequenceID(); + deletesInRAM.addDeleteTerms(terms, sequenceID, numDocumentsWriterPerThreads); + return sequenceID; + } finally { + sequenceIDLock.unlock(); + } + } + + long bufferDeleteTerm(final Term term) throws IOException { + sequenceIDLock.lock(); + try { + ensureOpen(); + final long sequenceID = nextSequenceID(); + deletesInRAM.addDeleteTerm(term, sequenceID, numDocumentsWriterPerThreads); + return sequenceID; + } finally { + sequenceIDLock.unlock(); + } + } + + long bufferDeleteQueries(final Query[] queries) throws IOException { + sequenceIDLock.lock(); + try { + ensureOpen(); + final long sequenceID = nextSequenceID(); + for (Query q : queries) { + deletesInRAM.addDeleteQuery(q, sequenceID, numDocumentsWriterPerThreads); + } + return sequenceID; + } finally { + sequenceIDLock.unlock(); + } + } + + long bufferDeleteQuery(final Query query) throws IOException { + sequenceIDLock.lock(); + try { + ensureOpen(); + final long sequenceID = nextSequenceID(); + deletesInRAM.addDeleteQuery(query, sequenceID, numDocumentsWriterPerThreads); + return sequenceID; + } finally { + sequenceIDLock.unlock(); + } + } + + private final void updateFlushedSequenceID() { + long newFlushedID = Long.MAX_VALUE; + for (long minSeqIDPerThread : minSequenceIDsPerThread.values()) { + if (minSeqIDPerThread < newFlushedID) { + newFlushedID = minSeqIDPerThread; + } + } + + this.flushedSequenceID = newFlushedID; + } + + final boolean flushAllThreads(final boolean flushDocStores, final boolean flushDeletes) + throws IOException { + return threadPool.executeAllThreads(new DocumentsWriterThreadPool.AllThreadsTask() { + @Override + public Boolean process(Iterator threadsIterator) throws IOException { + boolean anythingFlushed = false; + + if (flushDeletes) { + synchronized (indexWriter) { + if (applyDeletes(indexWriter.segmentInfos)) { + indexWriter.checkpoint(); + } + } + } + + while (threadsIterator.hasNext()) { + boolean perThreadFlushDocStores = flushDocStores; + DocumentsWriterPerThread perThread = threadsIterator.next(); + final int numDocs = perThread.getNumDocsInRAM(); + + // Always flush docs if there are any + boolean flushDocs = numDocs > 0; + + String docStoreSegment = perThread.getDocStoreSegment(); + if (docStoreSegment == null) { + perThreadFlushDocStores = false; + } + int docStoreOffset = perThread.getDocStoreOffset(); + boolean docStoreIsCompoundFile = false; + if (perThreadFlushDocStores + && (!flushDocs || !perThread.getSegment().equals(perThread.getDocStoreSegment()))) { + // We must separately flush the doc store + if (infoStream != null) { + message(" flush shared docStore segment " + docStoreSegment); + } + docStoreIsCompoundFile = flushDocStores(perThread); + flushDocStores(perThread); + perThreadFlushDocStores = false; + } + + String segment = perThread.getSegment(); + + // If we are flushing docs, segment must not be null: + assert segment != null || !flushDocs; + + if (flushDocs) { + SegmentInfo newSegment = perThread.flush(perThreadFlushDocStores); + + if (newSegment != null) { + anythingFlushed = true; + + if (0 == docStoreOffset && perThreadFlushDocStores) { + // This means we are flushing private doc stores + // with this segment, so it will not be shared + // with other segments + assert docStoreSegment != null; + assert docStoreSegment.equals(segment); + docStoreOffset = -1; + docStoreSegment = null; + docStoreIsCompoundFile = false; + } + newSegment.setDocStore(docStoreOffset, docStoreSegment, docStoreIsCompoundFile); + + IndexWriter.setDiagnostics(newSegment, "flush"); + finishFlushedSegment(newSegment, perThread); + } + } + } + + if (anythingFlushed) { + clearThreadBindings(); + + sequenceIDLock.lock(); + try { + flushedSequenceID = sequenceID; + } finally { + sequenceIDLock.unlock(); + } + numDocsInRAM.set(0); + } + + if (flushDeletes) { + deletesInRAM.clear(); + } + + + return anythingFlushed; + } + }); } /** Build compound file for the segment we just flushed */ - void createCompoundFile(String segment) throws IOException { + void createCompoundFile(String segment, DocumentsWriterPerThread perThread) throws IOException { CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); - for(String fileName : flushState.flushedFiles) { + for(String fileName : perThread.flushState.flushedFiles) { cfsWriter.addFile(fileName); } @@ -680,927 +357,395 @@ final class DocumentsWriter { cfsWriter.close(); } - /** Set flushPending if it is not already set and returns - * whether it was set. This is used by IndexWriter to - * trigger a single flush even when multiple threads are - * trying to do so. */ - synchronized boolean setFlushPending() { - if (flushPending) - return false; - else { - flushPending = true; - return true; - } - } - - synchronized void clearFlushPending() { - bufferIsFull = false; - flushPending = false; - } - - synchronized void pushDeletes() { - deletesFlushed.update(deletesInRAM); - } - - synchronized void close() { - closed = true; - notifyAll(); - } - - synchronized void initSegmentName(boolean onlyDocStore) { - if (segment == null && (!onlyDocStore || docStoreSegment == null)) { - segment = writer.newSegmentName(); - assert numDocsInRAM == 0; - } - if (docStoreSegment == null) { - docStoreSegment = segment; - assert numDocsInStore == 0; - } - } - - /** Returns a free (idle) ThreadState that may be used for - * indexing this one document. This call also pauses if a - * flush is pending. If delTerm is non-null then we - * buffer this deleted term after the thread state has - * been acquired. */ - synchronized DocumentsWriterThreadState getThreadState(Document doc, Term delTerm) throws IOException { - - // First, find a thread state. If this thread already - // has affinity to a specific ThreadState, use that one - // again. - DocumentsWriterThreadState state = threadBindings.get(Thread.currentThread()); - if (state == null) { - - // First time this thread has called us since last - // flush. Find the least loaded thread state: - DocumentsWriterThreadState minThreadState = null; - for(int i=0;i= maxThreadStates)) { - state = minThreadState; - state.numThreads++; - } else { - // Just create a new "private" thread state - DocumentsWriterThreadState[] newArray = new DocumentsWriterThreadState[1+threadStates.length]; - if (threadStates.length > 0) - System.arraycopy(threadStates, 0, newArray, 0, threadStates.length); - state = newArray[threadStates.length] = new DocumentsWriterThreadState(this); - threadStates = newArray; - } - threadBindings.put(Thread.currentThread(), state); - } - - // Next, wait until my thread state is idle (in case - // it's shared with other threads) and for threads to - // not be paused nor a flush pending: - waitReady(state); - - // Allocate segment name if this is the first doc since - // last flush: - initSegmentName(false); - - state.isIdle = false; - - boolean success = false; - try { - state.docState.docID = nextDocID; - - assert writer.testPoint("DocumentsWriter.ThreadState.init start"); - - if (delTerm != null) { - addDeleteTerm(delTerm, state.docState.docID); - state.doFlushAfter = timeToFlushDeletes(); - } - - assert writer.testPoint("DocumentsWriter.ThreadState.init after delTerm"); - - nextDocID++; - numDocsInRAM++; - - // We must at this point commit to flushing to ensure we - // always get N docs when we flush by doc count, even if - // > 1 thread is adding documents: - if (!flushPending && - maxBufferedDocs != IndexWriterConfig.DISABLE_AUTO_FLUSH - && numDocsInRAM >= maxBufferedDocs) { - flushPending = true; - state.doFlushAfter = true; - } - - success = true; - } finally { - if (!success) { - // Forcefully idle this ThreadState: - state.isIdle = true; - notifyAll(); - if (state.doFlushAfter) { - state.doFlushAfter = false; - flushPending = false; - } - } - } - - return state; - } - - /** Returns true if the caller (IndexWriter) should now - * flush. */ - boolean addDocument(Document doc, Analyzer analyzer) - throws CorruptIndexException, IOException { - return updateDocument(doc, analyzer, null); - } - - boolean updateDocument(Term t, Document doc, Analyzer analyzer) - throws CorruptIndexException, IOException { - return updateDocument(doc, analyzer, t); - } - - boolean updateDocument(Document doc, Analyzer analyzer, Term delTerm) - throws CorruptIndexException, IOException { - - // This call is synchronized but fast - final DocumentsWriterThreadState state = getThreadState(doc, delTerm); - - final DocState docState = state.docState; - docState.doc = doc; - docState.analyzer = analyzer; - - boolean success = false; - try { - // This call is not synchronized and does all the - // work - final DocWriter perDoc; + // nocommit + void finishFlushedSegment(SegmentInfo newSegment, DocumentsWriterPerThread perThread) throws IOException { + synchronized(indexWriter) { + indexWriter.segmentInfos.add(newSegment); + indexWriter.checkpoint(); + SegmentReader reader = indexWriter.readerPool.get(newSegment, false); + boolean any = false; try { - perDoc = state.consumer.processDocument(); + any = applyDeletes(reader, newSegment.getMinSequenceID(), newSegment.getMaxSequenceID(), perThread.sequenceIDs); } finally { - docState.clear(); + indexWriter.readerPool.release(reader); } - - // This call is synchronized but fast - finishDocument(state, perDoc); - - success = true; - } finally { - if (!success) { - synchronized(this) { - - if (aborting) { - state.isIdle = true; - notifyAll(); - abort(); - } else { - skipDocWriter.docID = docState.docID; - boolean success2 = false; - try { - waitQueue.add(skipDocWriter); - success2 = true; - } finally { - if (!success2) { - state.isIdle = true; - notifyAll(); - abort(); - return false; - } + if (any) { + indexWriter.checkpoint(); + } + + if (indexWriter.mergePolicy.useCompoundFile(indexWriter.segmentInfos, newSegment)) { + // Now build compound file + boolean success = false; + try { + createCompoundFile(newSegment.name, perThread); + success = true; + } finally { + if (!success) { + if (infoStream != null) { + message("hit exception " + + "reating compound file for newly flushed segment " + newSegment.name); } - - state.isIdle = true; - notifyAll(); - - // If this thread state had decided to flush, we - // must clear it so another thread can flush - if (state.doFlushAfter) { - state.doFlushAfter = false; - flushPending = false; - notifyAll(); - } - - // Immediately mark this document as deleted - // since likely it was partially added. This - // keeps indexing as "all or none" (atomic) when - // adding a document: - addDeleteDocID(state.docState.docID); + indexWriter.deleter.deleteFile(IndexFileNames.segmentFileName(newSegment.name, "", + IndexFileNames.COMPOUND_FILE_EXTENSION)); } } + + newSegment.setUseCompoundFile(true); + indexWriter.checkpoint(); } } - - return state.doFlushAfter || timeToFlushDeletes(); } - // for testing - synchronized int getNumBufferedDeleteTerms() { - return deletesInRAM.numTerms; - } - - // for testing - synchronized Map getBufferedDeleteTerms() { - return deletesInRAM.terms; - } - - /** Called whenever a merge has completed and the merged segments had deletions */ - synchronized void remapDeletes(SegmentInfos infos, int[][] docMaps, int[] delCounts, MergePolicy.OneMerge merge, int mergeDocCount) { - if (docMaps == null) - // The merged segments had no deletes so docIDs did not change and we have nothing to do - return; - MergeDocIDRemapper mapper = new MergeDocIDRemapper(infos, docMaps, delCounts, merge, mergeDocCount); - deletesInRAM.remap(mapper, infos, docMaps, delCounts, merge, mergeDocCount); - deletesFlushed.remap(mapper, infos, docMaps, delCounts, merge, mergeDocCount); - flushedDocCount -= mapper.docShift; - } - - synchronized private void waitReady(DocumentsWriterThreadState state) { - - while (!closed && ((state != null && !state.isIdle) || pauseThreads != 0 || flushPending || aborting)) { + + private boolean flushDocStores(DocumentsWriterPerThread perThread) throws IOException { + boolean useCompoundDocStore = false; + + String docStoreSegment; + + boolean success = false; try { - wait(); - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - } - - if (closed) - throw new AlreadyClosedException("this IndexWriter is closed"); - } - - boolean bufferDeleteTerms(Term[] terms) throws IOException { - synchronized(this) { - waitReady(null); - for (int i = 0; i < terms.length; i++) - addDeleteTerm(terms[i], numDocsInRAM); - } - return timeToFlushDeletes(); - } - - boolean bufferDeleteTerm(Term term) throws IOException { - synchronized(this) { - waitReady(null); - addDeleteTerm(term, numDocsInRAM); - } - return timeToFlushDeletes(); - } - - boolean bufferDeleteQueries(Query[] queries) throws IOException { - synchronized(this) { - waitReady(null); - for (int i = 0; i < queries.length; i++) - addDeleteQuery(queries[i], numDocsInRAM); - } - return timeToFlushDeletes(); - } - - boolean bufferDeleteQuery(Query query) throws IOException { - synchronized(this) { - waitReady(null); - addDeleteQuery(query, numDocsInRAM); - } - return timeToFlushDeletes(); - } - - synchronized boolean deletesFull() { - return (ramBufferSize != IndexWriterConfig.DISABLE_AUTO_FLUSH && - (deletesInRAM.bytesUsed + deletesFlushed.bytesUsed + numBytesUsed) >= ramBufferSize) || - (maxBufferedDeleteTerms != IndexWriterConfig.DISABLE_AUTO_FLUSH && - ((deletesInRAM.size() + deletesFlushed.size()) >= maxBufferedDeleteTerms)); - } - - synchronized boolean doApplyDeletes() { - // Very similar to deletesFull(), except we don't count - // numBytesUsed, because we are checking whether - // deletes (alone) are consuming too many resources now - // and thus should be applied. We apply deletes if RAM - // usage is > 1/2 of our allowed RAM buffer, to prevent - // too-frequent flushing of a long tail of tiny segments - // when merges (which always apply deletes) are - // infrequent. - return (ramBufferSize != IndexWriterConfig.DISABLE_AUTO_FLUSH && - (deletesInRAM.bytesUsed + deletesFlushed.bytesUsed) >= ramBufferSize/2) || - (maxBufferedDeleteTerms != IndexWriterConfig.DISABLE_AUTO_FLUSH && - ((deletesInRAM.size() + deletesFlushed.size()) >= maxBufferedDeleteTerms)); - } - - private boolean timeToFlushDeletes() { - balanceRAM(); - synchronized(this) { - return (bufferIsFull || deletesFull()) && setFlushPending(); - } - } - - void setMaxBufferedDeleteTerms(int maxBufferedDeleteTerms) { - this.maxBufferedDeleteTerms = maxBufferedDeleteTerms; - } - - int getMaxBufferedDeleteTerms() { - return maxBufferedDeleteTerms; - } - - synchronized boolean hasDeletes() { - return deletesFlushed.any(); - } - - synchronized boolean applyDeletes(SegmentInfos infos) throws IOException { - - if (!hasDeletes()) - return false; - - final long t0 = System.currentTimeMillis(); - - if (infoStream != null) - message("apply " + deletesFlushed.numTerms + " buffered deleted terms and " + - deletesFlushed.docIDs.size() + " deleted docIDs and " + - deletesFlushed.queries.size() + " deleted queries on " + - + infos.size() + " segments."); - - final int infosEnd = infos.size(); - - int docStart = 0; - boolean any = false; - for (int i = 0; i < infosEnd; i++) { - - // Make sure we never attempt to apply deletes to - // segment in external dir - assert infos.info(i).dir == directory; - - SegmentReader reader = writer.readerPool.get(infos.info(i), false); - try { - any |= applyDeletes(reader, docStart); - docStart += reader.maxDoc(); + docStoreSegment = perThread.closeDocStore(); + success = true; } finally { - writer.readerPool.release(reader); + if (!success && infoStream != null) { + message("hit exception closing doc store segment"); + } } - } - - deletesFlushed.clear(); - if (infoStream != null) { - message("apply deletes took " + (System.currentTimeMillis()-t0) + " msec"); - } - - return any; + + useCompoundDocStore = indexWriter.mergePolicy.useCompoundDocStore(indexWriter.segmentInfos); + + if (useCompoundDocStore && docStoreSegment != null && perThread.closedFiles().size() != 0) { + // Now build compound doc store file + + if (infoStream != null) { + message("create compound file " + + IndexFileNames.segmentFileName(docStoreSegment, "", + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION)); + } + + success = false; + + final int numSegments = indexWriter.segmentInfos.size(); + final String compoundFileName = IndexFileNames.segmentFileName(docStoreSegment, "", + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION); + + try { + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); + for (final String file : perThread.closedFiles()) { + cfsWriter.addFile(file); + } + + // Perform the merge + cfsWriter.close(); + success = true; + + } finally { + if (!success) { + if (infoStream != null) + message("hit exception building compound file doc store for segment " + docStoreSegment); + synchronized(indexWriter) { + indexWriter.deleter.deleteFile(compoundFileName); + } + abort(); + } + } + + synchronized(indexWriter) { + for (int i = 0; i < numSegments; i++) { + SegmentInfo si = indexWriter.segmentInfos.info(i); + if (si.getDocStoreOffset() != -1 && + si.getDocStoreSegment().equals(docStoreSegment)) + si.setDocStoreIsCompoundFile(true); + } + + indexWriter.checkpoint(); + + // In case the files we just merged into a CFS were + // not previously checkpointed: + indexWriter.deleter.deleteNewFiles(perThread.closedFiles()); + } + } + + return useCompoundDocStore; + + } + + // Returns true if an abort is in progress + void pauseAllThreads() { + threadPool.pauseAllThreads(); } - // used only by assert - private Term lastDeleteTerm; + void resumeAllThreads() { + threadPool.resumeAllThreads(); + } - // used only by assert - private boolean checkDeleteTerm(Term term) { - if (term != null) { - assert lastDeleteTerm == null || term.compareTo(lastDeleteTerm) > 0: "lastTerm=" + lastDeleteTerm + " vs term=" + term; + void close() { + sequenceIDLock.lock(); + try { + closed = true; + } finally { + sequenceIDLock.unlock(); + } + } + + private void ensureOpen() throws AlreadyClosedException { + if (closed) { + throw new AlreadyClosedException("this IndexWriter is closed"); + } + } + + private final boolean maybeFlushPerThread(DocumentsWriterPerThread perThread) throws IOException { + if (perThread.getNumDocsInRAM() == maxBufferedDocs) { + flushSegment(perThread, false); + assert perThread.getNumDocsInRAM() == 0; + return true; + } + + return false; + } + + private boolean flushSegment(DocumentsWriterPerThread perThread, boolean flushDocStores) + throws IOException { + if (perThread.getNumDocsInRAM() == 0 && !flushDocStores) { + return false; + } + + int docStoreOffset = perThread.getDocStoreOffset(); + String docStoreSegment = perThread.getDocStoreSegment(); + SegmentInfo newSegment = perThread.flush(flushDocStores); + + if (newSegment != null) { + newSegment.setDocStore(docStoreOffset, docStoreSegment, false); + finishFlushedSegment(newSegment, perThread); + return true; + } + return false; + } + + void abort() throws IOException { + threadPool.abort(); + try { + try { + abortedFiles = openFiles(); + } catch (Throwable t) { + abortedFiles = null; + } + + deletesInRAM.clear(); + // nocommit + // deletesFlushed.clear(); + + openFiles.clear(); + } finally { + threadPool.finishAbort(); + } + + } + + final List openFiles = new ArrayList(); + private Collection abortedFiles; // List of files that were written before last abort() + + /* + * Returns Collection of files in use by this instance, + * including any flushed segments. + */ + @SuppressWarnings("unchecked") + List openFiles() { + synchronized(openFiles) { + return (List) ((ArrayList) openFiles).clone(); + } + } + + + Collection abortedFiles() { + return abortedFiles; + } + + boolean hasDeletes() { + return deletesInRAM.hasDeletes(); + } + + // nocommit + int getNumDocsInRAM() { + return numDocsInRAM.get(); + } + + // nocommit + long getRAMUsed() { + return ramUsed.get(); + } + + // nocommit + // long getRAMUsed() { + // return numBytesUsed + deletesInRAM.bytesUsed + deletesFlushed.bytesUsed; + // } + + boolean applyDeletes(SegmentInfos infos) throws IOException { + synchronized(indexWriter) { + if (!hasDeletes()) + return false; + + final long t0 = System.currentTimeMillis(); + + if (infoStream != null) { + message("apply " + deletesInRAM.getNumDeletes() + " buffered deletes on " + + +infos.size() + " segments."); + } + + final int infosEnd = infos.size(); + + boolean any = false; + for (int i = 0; i < infosEnd; i++) { + + // Make sure we never attempt to apply deletes to + // segment in external dir + assert infos.info(i).dir == directory; + + SegmentInfo si = infos.info(i); + SegmentReader reader = indexWriter.readerPool.get(si, false); + try { + any |= applyDeletes(reader, si.getMinSequenceID(), si.getMaxSequenceID(), null); + } finally { + indexWriter.readerPool.release(reader); + } + } + + if (infoStream != null) { + message("apply deletes took " + (System.currentTimeMillis() - t0) + " msec"); + } + + return any; } - lastDeleteTerm = term; - return true; } // Apply buffered delete terms, queries and docIDs to the // provided reader - private final synchronized boolean applyDeletes(IndexReader reader, int docIDStart) - throws CorruptIndexException, IOException { + final boolean applyDeletes(IndexReader reader, long minSequenceID, long maxSequenceID, long[] sequenceIDs) + throws CorruptIndexException, IOException { + + assert sequenceIDs == null || sequenceIDs.length >= reader.maxDoc() : "reader.maxDoc=" + + reader.maxDoc() + ",sequenceIDs.length=" + sequenceIDs.length; - final int docEnd = docIDStart + reader.maxDoc(); boolean any = false; - assert checkDeleteTerm(null); - - // Delete by term - if (deletesFlushed.terms.size() > 0) { - Fields fields = reader.fields(); - if (fields == null) { - // This reader has no postings - return false; - } - - TermsEnum termsEnum = null; - - String currentField = null; - BytesRef termRef = new BytesRef(); - DocsEnum docs = null; - - for (Entry entry: deletesFlushed.terms.entrySet()) { - Term term = entry.getKey(); - // Since we visit terms sorted, we gain performance - // by re-using the same TermsEnum and seeking only - // forwards - if (term.field() != currentField) { - assert currentField == null || currentField.compareTo(term.field()) < 0; - currentField = term.field(); - Terms terms = fields.terms(currentField); - if (terms != null) { - termsEnum = terms.iterator(); - } else { - termsEnum = null; - } - } - - if (termsEnum == null) { - continue; - } - assert checkDeleteTerm(term); - - termRef.copy(term.text()); - - if (termsEnum.seek(termRef, false) == TermsEnum.SeekStatus.FOUND) { - DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs); - - if (docsEnum != null) { - docs = docsEnum; - int limit = entry.getValue().getNum(); - while (true) { - final int docID = docs.nextDoc(); - if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) { - break; - } - reader.deleteDocument(docID); - any = true; - } - } + // first: delete the documents that had non-aborting exceptions + if (sequenceIDs != null) { + for (int i = 0; i < reader.maxDoc(); i++) { + if (sequenceIDs[i] == -1) { + reader.deleteDocument(i); + any = true; } } } - - // Delete by docID - for (Integer docIdInt : deletesFlushed.docIDs) { - int docID = docIdInt.intValue(); - if (docID >= docIDStart && docID < docEnd) { - reader.deleteDocument(docID-docIDStart); - any = true; - } - } - - // Delete by query - if (deletesFlushed.queries.size() > 0) { + + if (deletesInRAM.hasDeletes()) { IndexSearcher searcher = new IndexSearcher(reader); - try { - for (Entry entry : deletesFlushed.queries.entrySet()) { - Query query = entry.getKey(); - int limit = entry.getValue().intValue(); - Weight weight = query.weight(searcher); - Scorer scorer = weight.scorer(reader, true, false); - if (scorer != null) { - while(true) { - int doc = scorer.nextDoc(); - if (((long) docIDStart) + doc >= limit) - break; - reader.deleteDocument(doc); - any = true; + + SortedMap deletes = deletesInRAM.deletes.getReadCopy(); + + SortedMap deleteTerms = new TreeMap(); + for (Entry entry : deletes.entrySet()) { + if (minSequenceID < entry.getKey()) { + BufferedDeletesInRAM.Delete delete = entry.getValue(); + if (delete instanceof BufferedDeletesInRAM.DeleteTerm) { + BufferedDeletesInRAM.DeleteTerm deleteTerm = (BufferedDeletesInRAM.DeleteTerm) delete; + deleteTerms.put(deleteTerm.term, entry.getKey()); + } else if (delete instanceof BufferedDeletesInRAM.DeleteTerms) { + BufferedDeletesInRAM.DeleteTerms terms = (BufferedDeletesInRAM.DeleteTerms) delete; + for (Term t : terms.terms) { + deleteTerms.put(t, entry.getKey()); + } + } else { + // delete query + BufferedDeletesInRAM.DeleteQuery deleteQuery = (BufferedDeletesInRAM.DeleteQuery) delete; + Query query = deleteQuery.query; + Weight weight = query.weight(searcher); + Scorer scorer = weight.scorer(reader, true, false); + if (scorer != null) { + while (true) { + int doc = scorer.nextDoc(); + if (doc == DocsEnum.NO_MORE_DOCS) { + break; + } + if ( (sequenceIDs != null && sequenceIDs[doc] < entry.getKey()) + || (sequenceIDs == null && maxSequenceID < entry.getKey())) { + reader.deleteDocument(doc); + any = true; + } + } + } + } + } + } + + // Delete by term + if (deleteTerms.size() > 0) { + Fields fields = reader.fields(); + if (fields == null) { + // This reader has no postings + return false; + } + + TermsEnum termsEnum = null; + + String currentField = null; + BytesRef termRef = new BytesRef(); + DocsEnum docs = null; + + for (Entry entry : deleteTerms.entrySet()) { + Term term = entry.getKey(); + // Since we visit terms sorted, we gain performance + // by re-using the same TermsEnum and seeking only + // forwards + if (term.field() != currentField) { + assert currentField == null || currentField.compareTo(term.field()) < 0; + currentField = term.field(); + Terms terms = fields.terms(currentField); + if (terms != null) { + termsEnum = terms.iterator(); + } else { + termsEnum = null; + } + } + + if (termsEnum == null) { + continue; + } + // assert checkDeleteTerm(term); + + termRef.copy(term.text()); + + if (termsEnum.seek(termRef, false) == TermsEnum.SeekStatus.FOUND) { + DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs); + + if (docsEnum != null) { + docs = docsEnum; + // int limit = entry.getValue().getNum(); + while (true) { + final int doc = docs.nextDoc(); + // if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) { + if (doc == DocsEnum.NO_MORE_DOCS) { + break; + } + if ( (sequenceIDs != null && sequenceIDs[doc] < entry.getValue()) + || (sequenceIDs == null && maxSequenceID < entry.getValue())) { + reader.deleteDocument(doc); + any = true; + } + } } } } - } finally { - searcher.close(); } } + return any; } - // Buffer a term in bufferedDeleteTerms, which records the - // current number of documents buffered in ram so that the - // delete term will be applied to those documents as well - // as the disk segments. - synchronized private void addDeleteTerm(Term term, int docCount) { - BufferedDeletes.Num num = deletesInRAM.terms.get(term); - final int docIDUpto = flushedDocCount + docCount; - if (num == null) - deletesInRAM.terms.put(term, new BufferedDeletes.Num(docIDUpto)); - else - num.setNum(docIDUpto); - deletesInRAM.numTerms++; - - deletesInRAM.addBytesUsed(BYTES_PER_DEL_TERM + term.text.length()*CHAR_NUM_BYTE); - } - - // Buffer a specific docID for deletion. Currently only - // used when we hit a exception when adding a document - synchronized private void addDeleteDocID(int docID) { - deletesInRAM.docIDs.add(Integer.valueOf(flushedDocCount+docID)); - deletesInRAM.addBytesUsed(BYTES_PER_DEL_DOCID); - } - - synchronized private void addDeleteQuery(Query query, int docID) { - deletesInRAM.queries.put(query, Integer.valueOf(flushedDocCount + docID)); - deletesInRAM.addBytesUsed(BYTES_PER_DEL_QUERY); - } - - /** Does the synchronized work to finish/flush the - * inverted document. */ - private void finishDocument(DocumentsWriterThreadState perThread, DocWriter docWriter) throws IOException { - - // Must call this w/o holding synchronized(this) else - // we'll hit deadlock: - balanceRAM(); - - synchronized(this) { - - assert docWriter == null || docWriter.docID == perThread.docState.docID; - - if (aborting) { - - // We are currently aborting, and another thread is - // waiting for me to become idle. We just forcefully - // idle this threadState; it will be fully reset by - // abort() - if (docWriter != null) - try { - docWriter.abort(); - } catch (Throwable t) { - } - - perThread.isIdle = true; - notifyAll(); - return; - } - - final boolean doPause; - - if (docWriter != null) - doPause = waitQueue.add(docWriter); - else { - skipDocWriter.docID = perThread.docState.docID; - doPause = waitQueue.add(skipDocWriter); - } - - if (doPause) - waitForWaitQueue(); - - if (bufferIsFull && !flushPending) { - flushPending = true; - perThread.doFlushAfter = true; - } - - perThread.isIdle = true; - notifyAll(); + void message(String message) { + if (infoStream != null) { + indexWriter.message("DW: " + message); } } - synchronized void waitForWaitQueue() { - do { - try { - wait(); - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - } while (!waitQueue.doResume()); - } - - private static class SkipDocWriter extends DocWriter { - @Override - void finish() { - } - @Override - void abort() { - } - @Override - long sizeInBytes() { - return 0; - } - } - final SkipDocWriter skipDocWriter = new SkipDocWriter(); - - long getRAMUsed() { - return numBytesUsed + deletesInRAM.bytesUsed + deletesFlushed.bytesUsed; - } - - long numBytesUsed; - - NumberFormat nf = NumberFormat.getInstance(); - - // Coarse estimates used to measure RAM usage of buffered deletes - final static int OBJECT_HEADER_BYTES = 8; - final static int POINTER_NUM_BYTE = Constants.JRE_IS_64BIT ? 8 : 4; - final static int INT_NUM_BYTE = 4; - final static int CHAR_NUM_BYTE = 2; - - /* Rough logic: HashMap has an array[Entry] w/ varying - load factor (say 2 * POINTER). Entry is object w/ Term - key, BufferedDeletes.Num val, int hash, Entry next - (OBJ_HEADER + 3*POINTER + INT). Term is object w/ - String field and String text (OBJ_HEADER + 2*POINTER). - We don't count Term's field since it's interned. - Term's text is String (OBJ_HEADER + 4*INT + POINTER + - OBJ_HEADER + string.length*CHAR). BufferedDeletes.num is - OBJ_HEADER + INT. */ - - final static int BYTES_PER_DEL_TERM = 8*POINTER_NUM_BYTE + 5*OBJECT_HEADER_BYTES + 6*INT_NUM_BYTE; - - /* Rough logic: del docIDs are List. Say list - allocates ~2X size (2*POINTER). Integer is OBJ_HEADER - + int */ - final static int BYTES_PER_DEL_DOCID = 2*POINTER_NUM_BYTE + OBJECT_HEADER_BYTES + INT_NUM_BYTE; - - /* Rough logic: HashMap has an array[Entry] w/ varying - load factor (say 2 * POINTER). Entry is object w/ - Query key, Integer val, int hash, Entry next - (OBJ_HEADER + 3*POINTER + INT). Query we often - undercount (say 24 bytes). Integer is OBJ_HEADER + INT. */ - final static int BYTES_PER_DEL_QUERY = 5*POINTER_NUM_BYTE + 2*OBJECT_HEADER_BYTES + 2*INT_NUM_BYTE + 24; - - /* Initial chunks size of the shared byte[] blocks used to - store postings data */ - final static int BYTE_BLOCK_SHIFT = 15; - final static int BYTE_BLOCK_SIZE = 1 << BYTE_BLOCK_SHIFT; - final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1; - final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK; - - /* if you increase this, you must fix field cache impl for - * getTerms/getTermsIndex requires <= 32768 */ - final static int MAX_TERM_LENGTH_UTF8 = BYTE_BLOCK_SIZE-2; - - private class ByteBlockAllocator extends ByteBlockPool.Allocator { - final int blockSize; - - ByteBlockAllocator(int blockSize) { - this.blockSize = blockSize; - } - - ArrayList freeByteBlocks = new ArrayList(); - - /* Allocate another byte[] from the shared pool */ - @Override - byte[] getByteBlock() { - synchronized(DocumentsWriter.this) { - final int size = freeByteBlocks.size(); - final byte[] b; - if (0 == size) { - b = new byte[blockSize]; - numBytesUsed += blockSize; - } else - b = freeByteBlocks.remove(size-1); - return b; - } - } - - /* Return byte[]'s to the pool */ - - @Override - void recycleByteBlocks(byte[][] blocks, int start, int end) { - synchronized(DocumentsWriter.this) { - for(int i=start;i blocks) { - synchronized(DocumentsWriter.this) { - final int size = blocks.size(); - for(int i=0;i freeIntBlocks = new ArrayList(); - - /* Allocate another int[] from the shared pool */ - synchronized int[] getIntBlock() { - final int size = freeIntBlocks.size(); - final int[] b; - if (0 == size) { - b = new int[INT_BLOCK_SIZE]; - numBytesUsed += INT_BLOCK_SIZE*INT_NUM_BYTE; - } else - b = freeIntBlocks.remove(size-1); - return b; - } - - synchronized void bytesUsed(long numBytes) { - numBytesUsed += numBytes; - } - - /* Return int[]s to the pool */ - synchronized void recycleIntBlocks(int[][] blocks, int start, int end) { - for(int i=start;i= ramBufferSize; - } - - if (doBalance) { - - if (infoStream != null) - message(" RAM: now balance allocations: usedMB=" + toMB(numBytesUsed) + - " vs trigger=" + toMB(ramBufferSize) + - " deletesMB=" + toMB(deletesRAMUsed) + - " byteBlockFree=" + toMB(byteBlockAllocator.freeByteBlocks.size()*BYTE_BLOCK_SIZE) + - " perDocFree=" + toMB(perDocAllocator.freeByteBlocks.size()*PER_DOC_BLOCK_SIZE)); - - final long startBytesUsed = numBytesUsed + deletesRAMUsed; - - int iter = 0; - - // We free equally from each pool in 32 KB - // chunks until we are below our threshold - // (freeLevel) - - boolean any = true; - - while(numBytesUsed+deletesRAMUsed > freeLevel) { - - synchronized(this) { - if (0 == perDocAllocator.freeByteBlocks.size() && - 0 == byteBlockAllocator.freeByteBlocks.size() && - 0 == freeIntBlocks.size() && !any) { - // Nothing else to free -- must flush now. - bufferIsFull = numBytesUsed+deletesRAMUsed > ramBufferSize; - if (infoStream != null) { - if (numBytesUsed+deletesRAMUsed > ramBufferSize) - message(" nothing to free; now set bufferIsFull"); - else - message(" nothing to free"); - } - break; - } - - if ((0 == iter % 4) && byteBlockAllocator.freeByteBlocks.size() > 0) { - byteBlockAllocator.freeByteBlocks.remove(byteBlockAllocator.freeByteBlocks.size()-1); - numBytesUsed -= BYTE_BLOCK_SIZE; - } - - if ((1 == iter % 4) && freeIntBlocks.size() > 0) { - freeIntBlocks.remove(freeIntBlocks.size()-1); - numBytesUsed -= INT_BLOCK_SIZE * INT_NUM_BYTE; - } - - if ((2 == iter % 4) && perDocAllocator.freeByteBlocks.size() > 0) { - // Remove upwards of 32 blocks (each block is 1K) - for (int i = 0; i < 32; ++i) { - perDocAllocator.freeByteBlocks.remove(perDocAllocator.freeByteBlocks.size() - 1); - numBytesUsed -= PER_DOC_BLOCK_SIZE; - if (perDocAllocator.freeByteBlocks.size() == 0) { - break; - } - } - } - } - - if ((3 == iter % 4) && any) - // Ask consumer to free any recycled state - any = consumer.freeRAM(); - - iter++; - } - - if (infoStream != null) - message(" after free: freedMB=" + nf.format((startBytesUsed-numBytesUsed-deletesRAMUsed)/1024./1024.) + " usedMB=" + nf.format((numBytesUsed+deletesRAMUsed)/1024./1024.)); - } - } - - final WaitQueue waitQueue = new WaitQueue(); - - private class WaitQueue { - DocWriter[] waiting; - int nextWriteDocID; - int nextWriteLoc; - int numWaiting; - long waitingBytes; - - public WaitQueue() { - waiting = new DocWriter[10]; - } - - synchronized void reset() { - // NOTE: nextWriteLoc doesn't need to be reset - assert numWaiting == 0; - assert waitingBytes == 0; - nextWriteDocID = 0; - } - - synchronized boolean doResume() { - return waitingBytes <= waitQueueResumeBytes; - } - - synchronized boolean doPause() { - return waitingBytes > waitQueuePauseBytes; - } - - synchronized void abort() { - int count = 0; - for(int i=0;i= nextWriteDocID; - - if (doc.docID == nextWriteDocID) { - writeDocument(doc); - while(true) { - doc = waiting[nextWriteLoc]; - if (doc != null) { - numWaiting--; - waiting[nextWriteLoc] = null; - waitingBytes -= doc.sizeInBytes(); - writeDocument(doc); - } else - break; - } - } else { - - // I finished before documents that were added - // before me. This can easily happen when I am a - // small doc and the docs before me were large, or, - // just due to luck in the thread scheduling. Just - // add myself to the queue and when that large doc - // finishes, it will flush me: - int gap = doc.docID - nextWriteDocID; - if (gap >= waiting.length) { - // Grow queue - DocWriter[] newArray = new DocWriter[ArrayUtil.oversize(gap, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - assert nextWriteLoc >= 0; - System.arraycopy(waiting, nextWriteLoc, newArray, 0, waiting.length-nextWriteLoc); - System.arraycopy(waiting, 0, newArray, waiting.length-nextWriteLoc, nextWriteLoc); - nextWriteLoc = 0; - waiting = newArray; - gap = doc.docID - nextWriteDocID; - } - - int loc = nextWriteLoc + gap; - if (loc >= waiting.length) - loc -= waiting.length; - - // We should only wrap one time - assert loc < waiting.length; - - // Nobody should be in my spot! - assert waiting[loc] == null; - waiting[loc] = doc; - numWaiting++; - waitingBytes += doc.sizeInBytes(); - } - - return doPause(); - } - } } diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index d2cda9cf3c6..2f53492e617 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -240,7 +240,7 @@ public class IndexWriter implements Closeable { * printed to infoStream, if set (see {@link * #setInfoStream}). */ - public final static int MAX_TERM_LENGTH = DocumentsWriter.MAX_TERM_LENGTH_UTF8; + public final static int MAX_TERM_LENGTH = DocumentsWriterRAMAllocator.MAX_TERM_LENGTH_UTF8; // The normal read buffer size defaults to 1024, but // increasing this during merging seems to yield @@ -271,10 +271,12 @@ public class IndexWriter implements Closeable { volatile SegmentInfos pendingCommit; // set when a commit is pending (after prepareCommit() & before commit()) volatile long pendingCommitChangeCount; - private SegmentInfos segmentInfos = new SegmentInfos(); // the segments + // nocommit - private + SegmentInfos segmentInfos = new SegmentInfos(); // the segments private DocumentsWriter docWriter; - private IndexFileDeleter deleter; + //nocommit - private + IndexFileDeleter deleter; private Set segmentsToOptimize = new HashSet(); // used by optimize to note those needing optimization @@ -289,8 +291,8 @@ public class IndexWriter implements Closeable { // Holds all SegmentInfo instances currently involved in // merges private HashSet mergingSegments = new HashSet(); - - private MergePolicy mergePolicy; + // nocommit - private + MergePolicy mergePolicy; // TODO 4.0: this should be made final once the setter is removed private /*final*/MergeScheduler mergeScheduler; private LinkedList pendingMerges = new LinkedList(); @@ -733,113 +735,6 @@ public class IndexWriter implements Closeable { throw new IllegalArgumentException("this method can only be called when the merge policy is the default LogMergePolicy"); } - /**

Get the current setting of whether newly flushed - * segments will use the compound file format. Note that - * this just returns the value previously set with - * setUseCompoundFile(boolean), or the default value - * (true). You cannot use this to query the status of - * previously flushed segments.

- * - *

Note that this method is a convenience method: it - * just calls mergePolicy.getUseCompoundFile as long as - * mergePolicy is an instance of {@link LogMergePolicy}. - * Otherwise an IllegalArgumentException is thrown.

- * - * @see #setUseCompoundFile(boolean) - * @deprecated use {@link LogMergePolicy#getUseCompoundDocStore()} and - * {@link LogMergePolicy#getUseCompoundFile()} directly. - */ - public boolean getUseCompoundFile() { - return getLogMergePolicy().getUseCompoundFile(); - } - - /** - *

- * Setting to turn on usage of a compound file. When on, multiple files for - * each segment are merged into a single file when a new segment is flushed. - *

- * - *

- * Note that this method is a convenience method: it just calls - * mergePolicy.setUseCompoundFile as long as mergePolicy is an instance of - * {@link LogMergePolicy}. Otherwise an IllegalArgumentException is thrown. - *

- * - * @deprecated use {@link LogMergePolicy#setUseCompoundDocStore(boolean)} and - * {@link LogMergePolicy#setUseCompoundFile(boolean)} directly. - * Note that this method set the given value on both, therefore - * you should consider doing the same. - */ - public void setUseCompoundFile(boolean value) { - getLogMergePolicy().setUseCompoundFile(value); - getLogMergePolicy().setUseCompoundDocStore(value); - } - - /** Expert: Set the Similarity implementation used by this IndexWriter. - * - * @see Similarity#setDefault(Similarity) - * @deprecated use {@link IndexWriterConfig#setSimilarity(Similarity)} instead - */ - public void setSimilarity(Similarity similarity) { - ensureOpen(); - this.similarity = similarity; - docWriter.setSimilarity(similarity); - // Required so config.getSimilarity returns the right value. But this will - // go away together with the method in 4.0. - config.setSimilarity(similarity); - } - - /** Expert: Return the Similarity implementation used by this IndexWriter. - * - *

This defaults to the current value of {@link Similarity#getDefault()}. - * @deprecated use {@link IndexWriterConfig#getSimilarity()} instead - */ - public Similarity getSimilarity() { - ensureOpen(); - return similarity; - } - - /** Expert: Set the interval between indexed terms. Large values cause less - * memory to be used by IndexReader, but slow random-access to terms. Small - * values cause more memory to be used by an IndexReader, and speed - * random-access to terms. - * - * This parameter determines the amount of computation required per query - * term, regardless of the number of documents that contain that term. In - * particular, it is the maximum number of other terms that must be - * scanned before a term is located and its frequency and position information - * may be processed. In a large index with user-entered query terms, query - * processing time is likely to be dominated not by term lookup but rather - * by the processing of frequency and positional data. In a small index - * or when many uncommon query terms are generated (e.g., by wildcard - * queries) term lookup may become a dominant cost. - * - * In particular, numUniqueTerms/interval terms are read into - * memory by an IndexReader, and, on average, interval/2 terms - * must be scanned for each random term access. - * - * @see #DEFAULT_TERM_INDEX_INTERVAL - * @deprecated use {@link IndexWriterConfig#setTermIndexInterval(int)} - */ - public void setTermIndexInterval(int interval) { - ensureOpen(); - this.termIndexInterval = interval; - // Required so config.getTermIndexInterval returns the right value. But this - // will go away together with the method in 4.0. - config.setTermIndexInterval(interval); - } - - /** Expert: Return the interval between indexed terms. - * - * @see #setTermIndexInterval(int) - * @deprecated use {@link IndexWriterConfig#getTermIndexInterval()} - */ - public int getTermIndexInterval() { - // We pass false because this method is called by SegmentMerger while we are in the process of closing - ensureOpen(false); - return termIndexInterval; - } - /** * Constructs an IndexWriter for the index in d. * Text will be analyzed with a. If create @@ -1028,7 +923,6 @@ public class IndexWriter implements Closeable { directory = d; analyzer = conf.getAnalyzer(); setMessageID(defaultInfoStream); - maxFieldLength = conf.getMaxFieldLength(); termIndexInterval = conf.getTermIndexInterval(); writeLockTimeout = conf.getWriteLockTimeout(); similarity = conf.getSimilarity(); @@ -1102,9 +996,10 @@ public class IndexWriter implements Closeable { setRollbackSegmentInfos(segmentInfos); - docWriter = new DocumentsWriter(directory, this, conf.getIndexingChain(), conf.getMaxThreadStates()); - docWriter.setInfoStream(infoStream); - docWriter.setMaxFieldLength(maxFieldLength); + docWriter = new DocumentsWriter(directory, this, conf); + // nocommit + //docWriter.setInfoStream(infoStream); + //docWriter.setMaxFieldLength(maxFieldLength); // Default deleter (for backwards compatibility) is // KeepOnlyLastCommitDeleter: @@ -1167,149 +1062,6 @@ public class IndexWriter implements Closeable { return config; } - /** - * Expert: set the merge policy used by this writer. - * - * @deprecated use {@link IndexWriterConfig#setMergePolicy(MergePolicy)} instead. - */ - public void setMergePolicy(MergePolicy mp) { - ensureOpen(); - if (mp == null) - throw new NullPointerException("MergePolicy must be non-null"); - - if (mergePolicy != mp) - mergePolicy.close(); - mergePolicy = mp; - mergePolicy.setIndexWriter(this); - pushMaxBufferedDocs(); - if (infoStream != null) - message("setMergePolicy " + mp); - // Required so config.getMergePolicy returns the right value. But this will - // go away together with the method in 4.0. - config.setMergePolicy(mp); - } - - /** - * Expert: returns the current MergePolicy in use by this writer. - * @see #setMergePolicy - * - * @deprecated use {@link IndexWriterConfig#getMergePolicy()} instead - */ - public MergePolicy getMergePolicy() { - ensureOpen(); - return mergePolicy; - } - - /** - * Expert: set the merge scheduler used by this writer. - * @deprecated use {@link IndexWriterConfig#setMergeScheduler(MergeScheduler)} instead - */ - synchronized public void setMergeScheduler(MergeScheduler mergeScheduler) throws CorruptIndexException, IOException { - ensureOpen(); - if (mergeScheduler == null) - throw new NullPointerException("MergeScheduler must be non-null"); - - if (this.mergeScheduler != mergeScheduler) { - finishMerges(true); - this.mergeScheduler.close(); - } - this.mergeScheduler = mergeScheduler; - if (infoStream != null) - message("setMergeScheduler " + mergeScheduler); - // Required so config.getMergeScheduler returns the right value. But this will - // go away together with the method in 4.0. - config.setMergeScheduler(mergeScheduler); - } - - /** - * Expert: returns the current MergeScheduler in use by this - * writer. - * @see #setMergeScheduler(MergeScheduler) - * @deprecated use {@link IndexWriterConfig#getMergeScheduler()} instead - */ - public MergeScheduler getMergeScheduler() { - ensureOpen(); - return mergeScheduler; - } - - /**

Determines the largest segment (measured by - * document count) that may be merged with other segments. - * Small values (e.g., less than 10,000) are best for - * interactive indexing, as this limits the length of - * pauses while indexing to a few seconds. Larger values - * are best for batched indexing and speedier - * searches.

- * - *

The default value is {@link Integer#MAX_VALUE}.

- * - *

Note that this method is a convenience method: it - * just calls mergePolicy.setMaxMergeDocs as long as - * mergePolicy is an instance of {@link LogMergePolicy}. - * Otherwise an IllegalArgumentException is thrown.

- * - *

The default merge policy ({@link - * LogByteSizeMergePolicy}) also allows you to set this - * limit by net size (in MB) of the segment, using {@link - * LogByteSizeMergePolicy#setMaxMergeMB}.

- * @deprecated use {@link LogMergePolicy#setMaxMergeDocs(int)} directly. - */ - public void setMaxMergeDocs(int maxMergeDocs) { - getLogMergePolicy().setMaxMergeDocs(maxMergeDocs); - } - - /** - *

Returns the largest segment (measured by document - * count) that may be merged with other segments.

- * - *

Note that this method is a convenience method: it - * just calls mergePolicy.getMaxMergeDocs as long as - * mergePolicy is an instance of {@link LogMergePolicy}. - * Otherwise an IllegalArgumentException is thrown.

- * - * @see #setMaxMergeDocs - * @deprecated use {@link LogMergePolicy#getMaxMergeDocs()} directly. - */ - public int getMaxMergeDocs() { - return getLogMergePolicy().getMaxMergeDocs(); - } - - /** - * The maximum number of terms that will be indexed for a single field in a - * document. This limits the amount of memory required for indexing, so that - * collections with very large files will not crash the indexing process by - * running out of memory. This setting refers to the number of running terms, - * not to the number of different terms.

- * Note: this silently truncates large documents, excluding from the - * index all terms that occur further in the document. If you know your source - * documents are large, be sure to set this value high enough to accomodate - * the expected size. If you set it to Integer.MAX_VALUE, then the only limit - * is your memory, but you should anticipate an OutOfMemoryError.

- * By default, no more than {@link #DEFAULT_MAX_FIELD_LENGTH} terms - * will be indexed for a field. - * @deprecated use {@link IndexWriterConfig#setMaxFieldLength(int)} instead - */ - public void setMaxFieldLength(int maxFieldLength) { - ensureOpen(); - this.maxFieldLength = maxFieldLength; - docWriter.setMaxFieldLength(maxFieldLength); - if (infoStream != null) - message("setMaxFieldLength " + maxFieldLength); - // Required so config.getMaxFieldLength returns the right value. But this - // will go away together with the method in 4.0. - config.setMaxFieldLength(maxFieldLength); - } - - /** - * Returns the maximum number of terms that will be - * indexed for a single field in a document. - * @see #setMaxFieldLength - * @deprecated use {@link IndexWriterConfig#getMaxFieldLength()} instead - */ - public int getMaxFieldLength() { - ensureOpen(); - return maxFieldLength; - } - /** Determines the minimal number of documents required * before the buffered in-memory documents are flushed as * a new Segment. Large values generally gives faster @@ -1543,7 +1295,8 @@ public class IndexWriter implements Closeable { public void setInfoStream(PrintStream infoStream) { ensureOpen(); setMessageID(infoStream); - docWriter.setInfoStream(infoStream); + // nocommit + //docWriter.setInfoStream(infoStream); deleter.setInfoStream(infoStream); if (infoStream != null) messageState(); @@ -1571,48 +1324,6 @@ public class IndexWriter implements Closeable { return infoStream != null; } - /** - * Sets the maximum time to wait for a write lock (in milliseconds) for this instance of IndexWriter. @see - * @see #setDefaultWriteLockTimeout to change the default value for all instances of IndexWriter. - * @deprecated use {@link IndexWriterConfig#setWriteLockTimeout(long)} instead - */ - public void setWriteLockTimeout(long writeLockTimeout) { - ensureOpen(); - this.writeLockTimeout = writeLockTimeout; - // Required so config.getWriteLockTimeout returns the right value. But this - // will go away together with the method in 4.0. - config.setWriteLockTimeout(writeLockTimeout); - } - - /** - * Returns allowed timeout when acquiring the write lock. - * @see #setWriteLockTimeout - * @deprecated use {@link IndexWriterConfig#getWriteLockTimeout()} - */ - public long getWriteLockTimeout() { - ensureOpen(); - return writeLockTimeout; - } - - /** - * Sets the default (for any instance of IndexWriter) maximum time to wait for a write lock (in - * milliseconds). - * @deprecated use {@link IndexWriterConfig#setDefaultWriteLockTimeout(long)} instead - */ - public static void setDefaultWriteLockTimeout(long writeLockTimeout) { - IndexWriterConfig.setDefaultWriteLockTimeout(writeLockTimeout); - } - - /** - * Returns default write lock timeout for newly - * instantiated IndexWriters. - * @see #setDefaultWriteLockTimeout - * @deprecated use {@link IndexWriterConfig#getDefaultWriteLockTimeout()} instead - */ - public static long getDefaultWriteLockTimeout() { - return IndexWriterConfig.getDefaultWriteLockTimeout(); - } - /** * Commits all changes to an index and closes all * associated files. Note that this may be a costly @@ -1774,8 +1485,9 @@ public class IndexWriter implements Closeable { closing = false; notifyAll(); if (!closed) { - if (docWriter != null) + if (docWriter != null) { docWriter.resumeAllThreads(); + } if (infoStream != null) message("hit exception while closing"); } @@ -1783,76 +1495,6 @@ public class IndexWriter implements Closeable { } } - /** Tells the docWriter to close its currently open shared - * doc stores (stored fields & vectors files). - * Return value specifices whether new doc store files are compound or not. - */ - private synchronized boolean flushDocStores() throws IOException { - - boolean useCompoundDocStore = false; - - String docStoreSegment; - - boolean success = false; - try { - docStoreSegment = docWriter.closeDocStore(); - success = true; - } finally { - if (!success && infoStream != null) { - message("hit exception closing doc store segment"); - } - } - - useCompoundDocStore = mergePolicy.useCompoundDocStore(segmentInfos); - - if (useCompoundDocStore && docStoreSegment != null && docWriter.closedFiles().size() != 0) { - // Now build compound doc store file - - if (infoStream != null) { - message("create compound file " + IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION)); - } - - success = false; - - final int numSegments = segmentInfos.size(); - final String compoundFileName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION); - - try { - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); - for (final String file : docWriter.closedFiles() ) { - cfsWriter.addFile(file); - } - - // Perform the merge - cfsWriter.close(); - success = true; - - } finally { - if (!success) { - if (infoStream != null) - message("hit exception building compound file doc store for segment " + docStoreSegment); - deleter.deleteFile(compoundFileName); - docWriter.abort(); - } - } - - for(int i=0;i - * Note that this effectively truncates large documents, excluding from the - * index terms that occur further in the document. If you know your source - * documents are large, be sure to set this value high enough to accommodate - * the expected size. If you set it to Integer.MAX_VALUE, then the only limit - * is your memory, but you should anticipate an OutOfMemoryError.

- * By default, no more than 10,000 terms will be indexed for a field. - * - * @see MaxFieldLength - */ - private int maxFieldLength; - /** * Adds a document to this index. If the document contains more than * {@link #setMaxFieldLength(int)} terms for a given field, the remainder are @@ -1972,8 +1598,8 @@ public class IndexWriter implements Closeable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addDocument(Document doc) throws CorruptIndexException, IOException { - addDocument(doc, analyzer); + public long addDocument(Document doc) throws CorruptIndexException, IOException { + return addDocument(doc, analyzer); } /** @@ -1993,36 +1619,36 @@ public class IndexWriter implements Closeable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { + public long addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { ensureOpen(); - boolean doFlush = false; boolean success = false; try { try { - doFlush = docWriter.addDocument(doc, analyzer); + long sequenceID = docWriter.addDocument(doc, analyzer); success = true; + return sequenceID; } finally { if (!success) { - - if (infoStream != null) + if (infoStream != null) { message("hit exception adding document"); - + } synchronized (this) { // If docWriter has some aborted files that were // never incref'd, then we clean them up here if (docWriter != null) { final Collection files = docWriter.abortedFiles(); - if (files != null) + if (files != null) { deleter.deleteNewFiles(files); + } } } } } - if (doFlush) - flush(true, false, false); } catch (OutOfMemoryError oom) { handleOOM(oom, "addDocument"); } + + return -1; } /** @@ -2036,15 +1662,14 @@ public class IndexWriter implements Closeable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Term term) throws CorruptIndexException, IOException { + public long deleteDocuments(Term term) throws CorruptIndexException, IOException { ensureOpen(); try { - boolean doFlush = docWriter.bufferDeleteTerm(term); - if (doFlush) - flush(true, false, false); + return docWriter.bufferDeleteTerm(term); } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteDocuments(Term)"); } + return -1; } /** @@ -2060,15 +1685,14 @@ public class IndexWriter implements Closeable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Term... terms) throws CorruptIndexException, IOException { + public long deleteDocuments(Term... terms) throws CorruptIndexException, IOException { ensureOpen(); try { - boolean doFlush = docWriter.bufferDeleteTerms(terms); - if (doFlush) - flush(true, false, false); + return docWriter.bufferDeleteTerms(terms); } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteDocuments(Term..)"); } + return -1; } /** @@ -2082,11 +1706,9 @@ public class IndexWriter implements Closeable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Query query) throws CorruptIndexException, IOException { + public long deleteDocuments(Query query) throws CorruptIndexException, IOException { ensureOpen(); - boolean doFlush = docWriter.bufferDeleteQuery(query); - if (doFlush) - flush(true, false, false); + return docWriter.bufferDeleteQuery(query); } /** @@ -2102,11 +1724,9 @@ public class IndexWriter implements Closeable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void deleteDocuments(Query... queries) throws CorruptIndexException, IOException { + public long deleteDocuments(Query... queries) throws CorruptIndexException, IOException { ensureOpen(); - boolean doFlush = docWriter.bufferDeleteQueries(queries); - if (doFlush) - flush(true, false, false); + return docWriter.bufferDeleteQueries(queries); } /** @@ -2149,35 +1769,37 @@ public class IndexWriter implements Closeable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void updateDocument(Term term, Document doc, Analyzer analyzer) + public long updateDocument(Term term, Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { ensureOpen(); try { - boolean doFlush = false; boolean success = false; try { - doFlush = docWriter.updateDocument(term, doc, analyzer); + long sequenceID = docWriter.updateDocument(term, doc, analyzer); success = true; + return sequenceID; } finally { if (!success) { - if (infoStream != null) + if (infoStream != null) { message("hit exception updating document"); + } synchronized (this) { // If docWriter has some aborted files that were // never incref'd, then we clean them up here final Collection files = docWriter.abortedFiles(); - if (files != null) + if (files != null) { deleter.deleteNewFiles(files); + } } } } - if (doFlush) - flush(true, false, false); } catch (OutOfMemoryError oom) { handleOOM(oom, "updateDocument"); } + + return -1; } // for test purpose @@ -2697,7 +2319,8 @@ public class IndexWriter implements Closeable { // Remove any buffered docs docWriter.abort(); - docWriter.setFlushedDocCount(0); + // nocommit + //docWriter.setFlushedDocCount(0); // Remove all segments segmentInfos.clear(); @@ -2790,7 +2413,8 @@ public class IndexWriter implements Closeable { * the index files referenced exist (correctly) in the * index directory. */ - private synchronized void checkpoint() throws IOException { + // nocommit - private + synchronized void checkpoint() throws IOException { changeCount++; deleter.checkpoint(segmentInfos, false); } @@ -2841,7 +2465,7 @@ public class IndexWriter implements Closeable { * starting index (see {@link #optimize()} for details). * *

- * NOTE: this method only copies the segments of the incomning indexes + * NOTE: this method only copies the segments of the incoming indexes * and does not merge them. Therefore deleted documents are not removed and * the new segments are not merged with the existing ones. Also, the segments * are copied as-is, meaning they are not converted to CFS if they aren't, @@ -2925,7 +2549,8 @@ public class IndexWriter implements Closeable { ensureOpen(); segmentInfos.addAll(infos); // Notify DocumentsWriter that the flushed count just increased - docWriter.updateFlushedDocCount(docCount); + // nocommit + //docWriter.updateFlushedDocCount(docCount); checkpoint(); } @@ -2977,11 +2602,12 @@ public class IndexWriter implements Closeable { checkpoint(); // Notify DocumentsWriter that the flushed count just increased - docWriter.updateFlushedDocCount(docCount); + // nocommit + //docWriter.updateFlushedDocCount(docCount); } // Now create the compound file if needed - if (mergePolicy instanceof LogMergePolicy && getUseCompoundFile()) { + if (mergePolicy instanceof LogMergePolicy && getLogMergePolicy().getUseCompoundFile()) { List files = null; @@ -3211,183 +2837,17 @@ public class IndexWriter implements Closeable { // synchronized, ie, merges should be allowed to commit // even while a flush is happening private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException { - try { - try { - return doFlushInternal(flushDocStores, flushDeletes); - } finally { - docWriter.balanceRAM(); - } - } finally { - docWriter.clearFlushPending(); - } - } - - // TODO: this method should not have to be entirely - // synchronized, ie, merges should be allowed to commit - // even while a flush is happening - private synchronized final boolean doFlushInternal(boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException { - - if (hitOOM) { - throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot flush"); - } - - ensureOpen(false); - - assert testPoint("startDoFlush"); - - doBeforeFlush(); - - flushCount++; - - // If we are flushing because too many deletes - // accumulated, then we should apply the deletes to free - // RAM: - flushDeletes |= docWriter.doApplyDeletes(); - - // Make sure no threads are actively adding a document. - // Returns true if docWriter is currently aborting, in - // which case we skip flushing this segment - if (infoStream != null) { - message("flush: now pause all indexing threads"); - } - if (docWriter.pauseAllThreads()) { - docWriter.resumeAllThreads(); - return false; - } - - try { - - SegmentInfo newSegment = null; - - final int numDocs = docWriter.getNumDocsInRAM(); - - // Always flush docs if there are any - boolean flushDocs = numDocs > 0; - - String docStoreSegment = docWriter.getDocStoreSegment(); - - assert docStoreSegment != null || numDocs == 0: "dss=" + docStoreSegment + " numDocs=" + numDocs; - - if (docStoreSegment == null) - flushDocStores = false; - - int docStoreOffset = docWriter.getDocStoreOffset(); - - boolean docStoreIsCompoundFile = false; - - if (infoStream != null) { - message(" flush: segment=" + docWriter.getSegment() + - " docStoreSegment=" + docWriter.getDocStoreSegment() + - " docStoreOffset=" + docStoreOffset + - " flushDocs=" + flushDocs + - " flushDeletes=" + flushDeletes + - " flushDocStores=" + flushDocStores + - " numDocs=" + numDocs + - " numBufDelTerms=" + docWriter.getNumBufferedDeleteTerms()); - message(" index before flush " + segString()); - } - - // Check if the doc stores must be separately flushed - // because other segments, besides the one we are about - // to flush, reference it - if (flushDocStores && (!flushDocs || !docWriter.getSegment().equals(docWriter.getDocStoreSegment()))) { - // We must separately flush the doc store - if (infoStream != null) - message(" flush shared docStore segment " + docStoreSegment); - - docStoreIsCompoundFile = flushDocStores(); - flushDocStores = false; - } - - String segment = docWriter.getSegment(); - - // If we are flushing docs, segment must not be null: - assert segment != null || !flushDocs; - - if (flushDocs) { - - boolean success = false; - final int flushedDocCount; - - try { - flushedDocCount = docWriter.flush(flushDocStores); - success = true; - } finally { - if (!success) { - if (infoStream != null) - message("hit exception flushing segment " + segment); - deleter.refresh(segment); - } - } - - if (0 == docStoreOffset && flushDocStores) { - // This means we are flushing private doc stores - // with this segment, so it will not be shared - // with other segments - assert docStoreSegment != null; - assert docStoreSegment.equals(segment); - docStoreOffset = -1; - docStoreIsCompoundFile = false; - docStoreSegment = null; - } - - // Create new SegmentInfo, but do not add to our - // segmentInfos until deletes are flushed - // successfully. - newSegment = new SegmentInfo(segment, - flushedDocCount, - directory, false, docStoreOffset, - docStoreSegment, docStoreIsCompoundFile, - docWriter.hasProx(), - docWriter.getCodec()); - - setDiagnostics(newSegment, "flush"); - } - - docWriter.pushDeletes(); - - if (flushDocs) { - segmentInfos.add(newSegment); - checkpoint(); - } - - if (flushDocs && mergePolicy.useCompoundFile(segmentInfos, newSegment)) { - // Now build compound file - boolean success = false; - try { - docWriter.createCompoundFile(segment); - success = true; - } finally { - if (!success) { - if (infoStream != null) - message("hit exception creating compound file for newly flushed segment " + segment); - deleter.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); - } - } - - newSegment.setUseCompoundFile(true); - checkpoint(); - } - - if (flushDeletes) { - applyDeletes(); - } - - if (flushDocs) - checkpoint(); - - doAfterFlush(); - - return flushDocs; - - } catch (OutOfMemoryError oom) { - handleOOM(oom, "doFlush"); - // never hit - return false; - } finally { - docWriter.clearFlushPending(); - docWriter.resumeAllThreads(); - } + return docWriter.flushAllThreads(flushDocStores, flushDeletes); + // nocommit +// try { +// try { +// return doFlushInternal(flushDocStores, flushDeletes); +// } finally { +// docWriter.balanceRAM(); +// } +// } finally { +// docWriter.clearFlushPending(); +// } } /** Expert: Return the total size of all index files currently cached in memory. @@ -3535,7 +2995,8 @@ public class IndexWriter implements Closeable { final int start = ensureContiguousMerge(merge); commitMergedDeletes(merge, mergedReader); - docWriter.remapDeletes(segmentInfos, merger.getDocMaps(), merger.getDelCounts(), merge, mergedDocCount); + // nocommit + //docWriter.remapDeletes(segmentInfos, merger.getDocMaps(), merger.getDelCounts(), merge, mergedDocCount); setMergeDocStoreIsCompoundFile(merge); merge.info.setHasProx(merger.hasProx()); @@ -3749,7 +3210,8 @@ public class IndexWriter implements Closeable { boolean mergeDocStores = false; boolean doFlushDocStore = false; - final String currentDocStoreSegment = docWriter.getDocStoreSegment(); + // nocommit + //final String currentDocStoreSegment = docWriter.getDocStoreSegment(); // Test each segment to be merged: check if we need to // flush/merge doc stores @@ -3793,9 +3255,10 @@ public class IndexWriter implements Closeable { // If the segment is referencing the current "live" // doc store outputs then we must merge - if (si.getDocStoreOffset() != -1 && currentDocStoreSegment != null && si.getDocStoreSegment().equals(currentDocStoreSegment)) { - doFlushDocStore = true; - } + // nocommit +// if (si.getDocStoreOffset() != -1 && currentDocStoreSegment != null && si.getDocStoreSegment().equals(currentDocStoreSegment)) { +// doFlushDocStore = true; +// } } final int docStoreOffset; @@ -3854,12 +3317,13 @@ public class IndexWriter implements Closeable { // CFS: mergingSegments.add(merge.info); } - - private void setDiagnostics(SegmentInfo info, String source) { + + // nocommit - private + static void setDiagnostics(SegmentInfo info, String source) { setDiagnostics(info, source, null); } - - private void setDiagnostics(SegmentInfo info, String source, Map details) { + + private static void setDiagnostics(SegmentInfo info, String source, Map details) { Map diagnostics = new HashMap(); diagnostics.put("source", source); diagnostics.put("lucene.version", Constants.LUCENE_VERSION); @@ -3964,7 +3428,7 @@ public class IndexWriter implements Closeable { // commit merged deletes SegmentReader reader = merge.readers[i] = readerPool.get(info, merge.mergeDocStores, MERGE_READ_BUFFER_SIZE, - -1); + -config.getReaderTermsIndexDivisor()); // We clone the segment readers because other // deletes may come in while we're merging so we @@ -4003,11 +3467,12 @@ public class IndexWriter implements Closeable { // readers will attempt to open an IndexInput // on files that have still-open IndexOutputs // against them: - if (dss.contains(docWriter.getDocStoreSegment())) { - if (infoStream != null) - message("now flush at mergeMiddle"); - doFlush(true, false); - } + // nocommit +// if (dss.contains(docWriter.getDocStoreSegment())) { +// if (infoStream != null) +// message("now flush at mergeMiddle"); +// doFlush(true, false); +// } } for(int i=0;i Date: Sat, 24 Jul 2010 06:04:30 +0000 Subject: [PATCH 005/200] LUCENE-2555: Remove shared doc stores git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@978805 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/index/DocConsumer.java | 4 +- .../apache/lucene/index/DocFieldConsumer.java | 6 +- .../lucene/index/DocFieldConsumers.java | 93 +------- .../lucene/index/DocFieldProcessor.java | 111 ++-------- .../index/DocFieldProcessorPerField.java | 13 ++ .../org/apache/lucene/index/DocInverter.java | 11 +- .../apache/lucene/index/DocumentsWriter.java | 132 ++--------- .../index/DocumentsWriterPerThread.java | 163 +------------- .../index/DocumentsWriterRAMAllocator.java | 1 - .../index/DocumentsWriterThreadPool.java | 2 +- .../org/apache/lucene/index/FieldsWriter.java | 9 +- .../lucene/index/FreqProxTermsWriter.java | 10 +- .../org/apache/lucene/index/IndexWriter.java | 192 ++-------------- .../lucene/index/InvertedDocConsumer.java | 5 +- .../lucene/index/InvertedDocEndConsumer.java | 1 - .../org/apache/lucene/index/MergePolicy.java | 4 - .../org/apache/lucene/index/NormsWriter.java | 4 - .../org/apache/lucene/index/SegmentInfo.java | 45 ++-- .../apache/lucene/index/SegmentMerger.java | 128 +++++------ .../lucene/index/SegmentWriteState.java | 8 +- .../lucene/index/StoredFieldsWriter.java | 207 ++++++------------ .../lucene/index/TermVectorsTermsWriter.java | 187 +++++----------- .../index/TermVectorsTermsWriterPerField.java | 28 +-- .../org/apache/lucene/index/TermsHash.java | 30 +-- .../lucene/index/TermsHashConsumer.java | 4 +- .../org/apache/lucene/index/TestCodecs.java | 6 +- .../index/TestConcurrentMergeScheduler.java | 2 +- .../test/org/apache/lucene/index/TestDoc.java | 2 +- .../apache/lucene/index/TestIndexWriter.java | 2 +- .../lucene/index/TestIndexWriterReader.java | 8 +- .../lucene/index/TestSegmentMerger.java | 4 +- .../index/codecs/preflex/TestSurrogates.java | 2 +- 32 files changed, 312 insertions(+), 1112 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocConsumer.java b/lucene/src/java/org/apache/lucene/index/DocConsumer.java index 92cb23a6c60..6783a3212f4 100644 --- a/lucene/src/java/org/apache/lucene/index/DocConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/DocConsumer.java @@ -20,9 +20,9 @@ package org.apache.lucene.index; import java.io.IOException; abstract class DocConsumer { - abstract DocumentsWriterPerThread.DocWriter processDocument() throws IOException; + abstract void processDocument() throws IOException; + abstract void finishDocument() throws IOException; abstract void flush(final SegmentWriteState state) throws IOException; - abstract void closeDocStore(final SegmentWriteState state) throws IOException; abstract void abort(); abstract boolean freeRAM(); } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java index d74de0862b4..f9951d381e0 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java @@ -27,10 +27,6 @@ abstract class DocFieldConsumer { /** Called when DocumentsWriter decides to create a new * segment */ abstract void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException; - - /** Called when DocumentsWriter decides to close the doc - * stores */ - abstract void closeDocStore(SegmentWriteState state) throws IOException; /** Called when an aborting exception is hit */ abstract void abort(); @@ -44,7 +40,7 @@ abstract class DocFieldConsumer { abstract DocFieldConsumerPerField addField(FieldInfo fi); - abstract DocumentsWriterPerThread.DocWriter finishDocument() throws IOException; + abstract void finishDocument() throws IOException; void setFieldInfos(FieldInfos fieldInfos) { this.fieldInfos = fieldInfos; diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java b/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java index 36241ee02ad..5f55a7a9282 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldConsumers.java @@ -21,9 +21,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.RamUsageEstimator; - /** This is just a "splitter" class: it lets you wrap two * DocFieldConsumer instances as a single consumer. */ @@ -61,15 +58,6 @@ final class DocFieldConsumers extends DocFieldConsumer { two.flush(twoFieldsToFlush, state); } - @Override - public void closeDocStore(SegmentWriteState state) throws IOException { - try { - one.closeDocStore(state); - } finally { - two.closeDocStore(state); - } - } - @Override public void abort() { try { @@ -86,83 +74,12 @@ final class DocFieldConsumers extends DocFieldConsumer { return any; } - PerDoc[] docFreeList = new PerDoc[1]; - int freeCount; - int allocCount; - - PerDoc getPerDoc() { - if (freeCount == 0) { - allocCount++; - if (allocCount > docFreeList.length) { - // Grow our free list up front to make sure we have - // enough space to recycle all outstanding PerDoc - // instances - assert allocCount == 1+docFreeList.length; - docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - } - return new PerDoc(); - } else - return docFreeList[--freeCount]; - } - - void freePerDoc(PerDoc perDoc) { - assert freeCount < docFreeList.length; - docFreeList[freeCount++] = perDoc; - } - - class PerDoc extends DocumentsWriterPerThread.DocWriter { - - DocumentsWriterPerThread.DocWriter writerOne; - DocumentsWriterPerThread.DocWriter writerTwo; - - @Override - public long sizeInBytes() { - return writerOne.sizeInBytes() + writerTwo.sizeInBytes(); - } - - @Override - public void finish() throws IOException { - try { - try { - writerOne.finish(); - } finally { - writerTwo.finish(); - } - } finally { - freePerDoc(this); - } - } - - @Override - public void abort() { - try { - try { - writerOne.abort(); - } finally { - writerTwo.abort(); - } - } finally { - freePerDoc(this); - } - } - } - @Override - public DocumentsWriterPerThread.DocWriter finishDocument() throws IOException { - final DocumentsWriterPerThread.DocWriter oneDoc = one.finishDocument(); - final DocumentsWriterPerThread.DocWriter twoDoc = two.finishDocument(); - if (oneDoc == null) - return twoDoc; - else if (twoDoc == null) - return oneDoc; - else { - DocFieldConsumers.PerDoc both = getPerDoc(); - both.docID = docState.docID; - assert oneDoc.docID == docState.docID; - assert twoDoc.docID == docState.docID; - both.writerOne = oneDoc; - both.writerTwo = twoDoc; - return both; + public void finishDocument() throws IOException { + try { + one.finishDocument(); + } finally { + two.finishDocument(); } } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java index 8ce56b834c4..50a357414e6 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java @@ -26,8 +26,6 @@ import java.util.Map; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.RamUsageEstimator; /** @@ -66,12 +64,6 @@ final class DocFieldProcessor extends DocConsumer { fieldsWriter = new StoredFieldsWriter(docWriter, fieldInfos); } - @Override - public void closeDocStore(SegmentWriteState state) throws IOException { - consumer.closeDocStore(state); - fieldsWriter.closeDocStore(state); - } - @Override public void flush(SegmentWriteState state) throws IOException { @@ -105,8 +97,11 @@ final class DocFieldProcessor extends DocConsumer { } } - fieldsWriter.abort(); - consumer.abort(); + try { + fieldsWriter.abort(); + } finally { + consumer.abort(); + } } @Override @@ -190,7 +185,7 @@ final class DocFieldProcessor extends DocConsumer { } @Override - public DocumentsWriterPerThread.DocWriter processDocument() throws IOException { + public void processDocument() throws IOException { consumer.startDocument(); fieldsWriter.startDocument(); @@ -259,14 +254,9 @@ final class DocFieldProcessor extends DocConsumer { fields[fieldCount++] = fp; fp.lastGen = thisFieldGen; } + + fp.addField(field); - if (fp.fieldCount == fp.fields.length) { - Fieldable[] newArray = new Fieldable[fp.fields.length*2]; - System.arraycopy(fp.fields, 0, newArray, 0, fp.fieldCount); - fp.fields = newArray; - } - - fp.fields[fp.fieldCount++] = field; if (field.isStored()) { fieldsWriter.addField(field, fp.fieldInfo); } @@ -287,24 +277,18 @@ final class DocFieldProcessor extends DocConsumer { docState.infoStream.println("WARNING: document contains at least one immense term (whose UTF8 encoding is longer than the max length " + DocumentsWriterRAMAllocator.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'"); docState.maxTermPrefix = null; } - - final DocumentsWriterPerThread.DocWriter one = fieldsWriter.finishDocument(); - final DocumentsWriterPerThread.DocWriter two = consumer.finishDocument(); - if (one == null) { - return two; - } else if (two == null) { - return one; - } else { - PerDoc both = getPerDoc(); - both.docID = docState.docID; - assert one.docID == docState.docID; - assert two.docID == docState.docID; - both.one = one; - both.two = two; - return both; + } + + @Override + void finishDocument() throws IOException { + try { + fieldsWriter.finishDocument(); + } finally { + consumer.finishDocument(); } } + void quickSort(DocFieldProcessorPerField[] array, int lo, int hi) { if (lo >= hi) return; @@ -365,65 +349,4 @@ final class DocFieldProcessor extends DocConsumer { quickSort(array, lo, left); quickSort(array, left + 1, hi); } - - PerDoc[] docFreeList = new PerDoc[1]; - int freeCount; - int allocCount; - - PerDoc getPerDoc() { - if (freeCount == 0) { - allocCount++; - if (allocCount > docFreeList.length) { - // Grow our free list up front to make sure we have - // enough space to recycle all outstanding PerDoc - // instances - assert allocCount == 1+docFreeList.length; - docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - } - return new PerDoc(); - } else - return docFreeList[--freeCount]; - } - - void freePerDoc(PerDoc perDoc) { - assert freeCount < docFreeList.length; - docFreeList[freeCount++] = perDoc; - } - - class PerDoc extends DocumentsWriterPerThread.DocWriter { - - DocumentsWriterPerThread.DocWriter one; - DocumentsWriterPerThread.DocWriter two; - - @Override - public long sizeInBytes() { - return one.sizeInBytes() + two.sizeInBytes(); - } - - @Override - public void finish() throws IOException { - try { - try { - one.finish(); - } finally { - two.finish(); - } - } finally { - freePerDoc(this); - } - } - - @Override - public void abort() { - try { - try { - one.abort(); - } finally { - two.abort(); - } - } finally { - freePerDoc(this); - } - } - } } diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java b/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java index 4e961efc444..2413d032baa 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java @@ -18,6 +18,8 @@ package org.apache.lucene.index; */ import org.apache.lucene.document.Fieldable; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.RamUsageEstimator; /** * Holds all per thread, per field state. @@ -39,6 +41,17 @@ final class DocFieldProcessorPerField { this.fieldInfo = fieldInfo; } + public void addField(Fieldable field) { + if (fieldCount == fields.length) { + int newSize = ArrayUtil.oversize(fieldCount + 1, RamUsageEstimator.NUM_BYTES_OBJ_REF); + Fieldable[] newArray = new Fieldable[newSize]; + System.arraycopy(fields, 0, newArray, 0, fieldCount); + fields = newArray; + } + + fields[fieldCount++] = field; + } + public void abort() { consumer.abort(); } diff --git a/lucene/src/java/org/apache/lucene/index/DocInverter.java b/lucene/src/java/org/apache/lucene/index/DocInverter.java index f34e234f163..e6f3109750b 100644 --- a/lucene/src/java/org/apache/lucene/index/DocInverter.java +++ b/lucene/src/java/org/apache/lucene/index/DocInverter.java @@ -95,18 +95,11 @@ final class DocInverter extends DocFieldConsumer { } @Override - public DocumentsWriterPerThread.DocWriter finishDocument() throws IOException { + public void finishDocument() throws IOException { // TODO: allow endConsumer.finishDocument to also return // a DocWriter endConsumer.finishDocument(); - return consumer.finishDocument(); - } - - - @Override - public void closeDocStore(SegmentWriteState state) throws IOException { - consumer.closeDocStore(state); - endConsumer.closeDocStore(state); + consumer.finishDocument(); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 712ca62724a..f368854ed50 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -130,7 +130,7 @@ final class DocumentsWriter { long updateDocument(final Term delTerm, final Document doc, final Analyzer analyzer) throws CorruptIndexException, IOException { - return threadPool.executePerThread(this, doc, + long seqID = threadPool.executePerThread(this, doc, new DocumentsWriterThreadPool.PerThreadTask() { @Override public Long process(final DocumentsWriterPerThread perThread) throws IOException { @@ -156,11 +156,14 @@ final class DocumentsWriter { if (finishAddDocument(perThread, perThreadRAMUsedBeforeAdd)) { super.clearThreadBindings(); - indexWriter.maybeMerge(); } return sequenceID; } }); + + indexWriter.maybeMerge(); + + return seqID; } private final boolean finishAddDocument(DocumentsWriterPerThread perThread, @@ -252,7 +255,7 @@ final class DocumentsWriter { this.flushedSequenceID = newFlushedID; } - final boolean flushAllThreads(final boolean flushDocStores, final boolean flushDeletes) + final boolean flushAllThreads(final boolean flushDeletes) throws IOException { return threadPool.executeAllThreads(new DocumentsWriterThreadPool.AllThreadsTask() { @Override @@ -260,61 +263,29 @@ final class DocumentsWriter { boolean anythingFlushed = false; if (flushDeletes) { - synchronized (indexWriter) { - if (applyDeletes(indexWriter.segmentInfos)) { - indexWriter.checkpoint(); - } + if (applyDeletes(indexWriter.segmentInfos)) { + indexWriter.checkpoint(); } } while (threadsIterator.hasNext()) { - boolean perThreadFlushDocStores = flushDocStores; DocumentsWriterPerThread perThread = threadsIterator.next(); final int numDocs = perThread.getNumDocsInRAM(); // Always flush docs if there are any boolean flushDocs = numDocs > 0; - String docStoreSegment = perThread.getDocStoreSegment(); - if (docStoreSegment == null) { - perThreadFlushDocStores = false; - } - int docStoreOffset = perThread.getDocStoreOffset(); - boolean docStoreIsCompoundFile = false; - if (perThreadFlushDocStores - && (!flushDocs || !perThread.getSegment().equals(perThread.getDocStoreSegment()))) { - // We must separately flush the doc store - if (infoStream != null) { - message(" flush shared docStore segment " + docStoreSegment); - } - docStoreIsCompoundFile = flushDocStores(perThread); - flushDocStores(perThread); - perThreadFlushDocStores = false; - } - String segment = perThread.getSegment(); // If we are flushing docs, segment must not be null: assert segment != null || !flushDocs; if (flushDocs) { - SegmentInfo newSegment = perThread.flush(perThreadFlushDocStores); + SegmentInfo newSegment = perThread.flush(); if (newSegment != null) { anythingFlushed = true; - if (0 == docStoreOffset && perThreadFlushDocStores) { - // This means we are flushing private doc stores - // with this segment, so it will not be shared - // with other segments - assert docStoreSegment != null; - assert docStoreSegment.equals(segment); - docStoreOffset = -1; - docStoreSegment = null; - docStoreIsCompoundFile = false; - } - newSegment.setDocStore(docStoreOffset, docStoreSegment, docStoreIsCompoundFile); - IndexWriter.setDiagnostics(newSegment, "flush"); finishFlushedSegment(newSegment, perThread); } @@ -361,6 +332,7 @@ final class DocumentsWriter { synchronized(indexWriter) { indexWriter.segmentInfos.add(newSegment); indexWriter.checkpoint(); + SegmentReader reader = indexWriter.readerPool.get(newSegment, false); boolean any = false; try { @@ -389,84 +361,15 @@ final class DocumentsWriter { } } - newSegment.setUseCompoundFile(true); - indexWriter.checkpoint(); - } - } - } - - - private boolean flushDocStores(DocumentsWriterPerThread perThread) throws IOException { - boolean useCompoundDocStore = false; - - String docStoreSegment; - - boolean success = false; - try { - docStoreSegment = perThread.closeDocStore(); - success = true; - } finally { - if (!success && infoStream != null) { - message("hit exception closing doc store segment"); - } - } - - useCompoundDocStore = indexWriter.mergePolicy.useCompoundDocStore(indexWriter.segmentInfos); - - if (useCompoundDocStore && docStoreSegment != null && perThread.closedFiles().size() != 0) { - // Now build compound doc store file - - if (infoStream != null) { - message("create compound file " - + IndexFileNames.segmentFileName(docStoreSegment, "", - IndexFileNames.COMPOUND_FILE_STORE_EXTENSION)); - } - - success = false; - - final int numSegments = indexWriter.segmentInfos.size(); - final String compoundFileName = IndexFileNames.segmentFileName(docStoreSegment, "", - IndexFileNames.COMPOUND_FILE_STORE_EXTENSION); - - try { - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); - for (final String file : perThread.closedFiles()) { - cfsWriter.addFile(file); - } - - // Perform the merge - cfsWriter.close(); - success = true; - - } finally { - if (!success) { - if (infoStream != null) - message("hit exception building compound file doc store for segment " + docStoreSegment); - synchronized(indexWriter) { - indexWriter.deleter.deleteFile(compoundFileName); - } - abort(); - } - } - synchronized(indexWriter) { - for (int i = 0; i < numSegments; i++) { - SegmentInfo si = indexWriter.segmentInfos.info(i); - if (si.getDocStoreOffset() != -1 && - si.getDocStoreSegment().equals(docStoreSegment)) - si.setDocStoreIsCompoundFile(true); - } - + newSegment.setUseCompoundFile(true); indexWriter.checkpoint(); - // In case the files we just merged into a CFS were // not previously checkpointed: indexWriter.deleter.deleteNewFiles(perThread.closedFiles()); } } - - return useCompoundDocStore; - + } } // Returns true if an abort is in progress @@ -495,7 +398,7 @@ final class DocumentsWriter { private final boolean maybeFlushPerThread(DocumentsWriterPerThread perThread) throws IOException { if (perThread.getNumDocsInRAM() == maxBufferedDocs) { - flushSegment(perThread, false); + flushSegment(perThread); assert perThread.getNumDocsInRAM() == 0; return true; } @@ -503,18 +406,15 @@ final class DocumentsWriter { return false; } - private boolean flushSegment(DocumentsWriterPerThread perThread, boolean flushDocStores) + private boolean flushSegment(DocumentsWriterPerThread perThread) throws IOException { - if (perThread.getNumDocsInRAM() == 0 && !flushDocStores) { + if (perThread.getNumDocsInRAM() == 0) { return false; } - int docStoreOffset = perThread.getDocStoreOffset(); - String docStoreSegment = perThread.getDocStoreSegment(); - SegmentInfo newSegment = perThread.flush(flushDocStores); + SegmentInfo newSegment = perThread.flush(); if (newSegment != null) { - newSegment.setDocStore(docStoreOffset, docStoreSegment, false); finishFlushedSegment(newSegment, perThread); return true; } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index fafd80b4289..7233463f7de 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -10,7 +10,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMFile; import org.apache.lucene.util.ArrayUtil; public class DocumentsWriterPerThread { @@ -94,10 +93,6 @@ public class DocumentsWriterPerThread { } catch (Throwable t) { } - docStoreSegment = null; - numDocsInStore = 0; - docStoreOffset = 0; - // Reset all postings data doAfterFlush(); @@ -121,18 +116,14 @@ public class DocumentsWriterPerThread { private DocFieldProcessor docFieldProcessor; String segment; // Current segment we are working on - private String docStoreSegment; // Current doc-store segment we are writing - private int docStoreOffset; // Current starting doc-store offset of current segment boolean aborting; // True if an abort is pending private final PrintStream infoStream; private int numDocsInRAM; - private int numDocsInStore; private int flushedDocCount; SegmentWriteState flushState; long[] sequenceIDs = new long[8]; - final List closedFiles = new ArrayList(); long numBytesUsed; @@ -161,13 +152,15 @@ public class DocumentsWriterPerThread { docState.doc = doc; docState.analyzer = analyzer; docState.docID = numDocsInRAM; - initSegmentName(false); + if (segment == null) { + // this call is synchronized on IndexWriter.segmentInfos + segment = writer.newSegmentName(); + assert numDocsInRAM == 0; + } - final DocWriter perDoc; - boolean success = false; try { - perDoc = consumer.processDocument(); + consumer.processDocument(); success = true; } finally { @@ -181,9 +174,7 @@ public class DocumentsWriterPerThread { success = false; try { - if (perDoc != null) { - perDoc.finish(); - } + consumer.finishDocument(); success = true; } finally { @@ -201,7 +192,6 @@ public class DocumentsWriterPerThread { sequenceIDs[numDocsInRAM] = sequenceID; numDocsInRAM++; - numDocsInStore++; } int getNumDocsInRAM() { @@ -226,26 +216,6 @@ public class DocumentsWriterPerThread { return flushState.codec; } - void initSegmentName(boolean onlyDocStore) { - if (segment == null && (!onlyDocStore || docStoreSegment == null)) { - // this call is synchronized on IndexWriter.segmentInfos - segment = writer.newSegmentName(); - assert numDocsInRAM == 0; - } - if (docStoreSegment == null) { - docStoreSegment = segment; - assert numDocsInStore == 0; - } - } - - - private void initFlushState(boolean onlyDocStore) { - initSegmentName(onlyDocStore); - flushState = new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos, - docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval(), - writer.codecs); - } - /** Reset after a flush */ private void doAfterFlush() throws IOException { segment = null; @@ -253,12 +223,12 @@ public class DocumentsWriterPerThread { } /** Flush all pending docs to a new segment */ - SegmentInfo flush(boolean closeDocStore) throws IOException { + SegmentInfo flush() throws IOException { assert numDocsInRAM > 0; - initFlushState(closeDocStore); - - docStoreOffset = numDocsInStore; + flushState = new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos, + numDocsInRAM, writer.getConfig().getTermIndexInterval(), + writer.codecs); if (infoStream != null) { message("flush postings as segment " + flushState.segmentName + " numDocs=" + numDocsInRAM); @@ -267,22 +237,12 @@ public class DocumentsWriterPerThread { boolean success = false; try { - - if (closeDocStore) { - assert flushState.docStoreSegmentName != null; - assert flushState.docStoreSegmentName.equals(flushState.segmentName); - closeDocStore(); - flushState.numDocsInStore = 0; - } - consumer.flush(flushState); if (infoStream != null) { SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, false, - docStoreOffset, flushState.docStoreSegmentName, - false, hasProx(), getCodec()); @@ -305,8 +265,6 @@ public class DocumentsWriterPerThread { SegmentInfo newSegment = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, false, - docStoreOffset, flushState.docStoreSegmentName, - false, hasProx(), getCodec()); @@ -325,62 +283,17 @@ public class DocumentsWriterPerThread { } } - /** Closes the current open doc stores an returns the doc - * store segment name. This returns null if there are * - * no buffered documents. */ - String closeDocStore() throws IOException { - - // nocommit -// if (infoStream != null) -// message("closeDocStore: " + openFiles.size() + " files to flush to segment " + docStoreSegment + " numDocs=" + numDocsInStore); - - boolean success = false; - - try { - initFlushState(true); - closedFiles.clear(); - - consumer.closeDocStore(flushState); - // nocommit - //assert 0 == openFiles.size(); - - String s = docStoreSegment; - docStoreSegment = null; - docStoreOffset = 0; - numDocsInStore = 0; - success = true; - return s; - } finally { - if (!success) { - parent.abort(); - } - } - } - - /** Get current segment name we are writing. */ String getSegment() { return segment; } - /** Returns the current doc store segment we are writing - * to. */ - String getDocStoreSegment() { - return docStoreSegment; - } - - /** Returns the doc offset into the shared doc store for - * the current buffered docs. */ - int getDocStoreOffset() { - return docStoreOffset; - } - - @SuppressWarnings("unchecked") List closedFiles() { return (List) ((ArrayList) closedFiles).clone(); } + void addOpenFile(String name) { synchronized(parent.openFiles) { assert !parent.openFiles.contains(name); @@ -396,58 +309,6 @@ public class DocumentsWriterPerThread { closedFiles.add(name); } - /** Consumer returns this on each doc. This holds any - * state that must be flushed synchronized "in docID - * order". We gather these and flush them in order. */ - abstract static class DocWriter { - DocWriter next; - int docID; - abstract void finish() throws IOException; - abstract void abort(); - abstract long sizeInBytes(); - - void setNext(DocWriter next) { - this.next = next; - } - } - - /** - * Create and return a new DocWriterBuffer. - */ - PerDocBuffer newPerDocBuffer() { - return new PerDocBuffer(); - } - - /** - * RAMFile buffer for DocWriters. - */ - class PerDocBuffer extends RAMFile { - - /** - * Allocate bytes used from shared pool. - */ - protected byte[] newBuffer(int size) { - assert size == DocumentsWriterRAMAllocator.PER_DOC_BLOCK_SIZE; - return ramAllocator.perDocAllocator.getByteBlock(); - } - - /** - * Recycle the bytes used. - */ - synchronized void recycle() { - if (buffers.size() > 0) { - setLength(0); - - // Recycle the blocks - ramAllocator.perDocAllocator.recycleByteBlocks(buffers); - buffers.clear(); - sizeInBytes = 0; - - assert numBuffers() == 0; - } - } - } - void bytesUsed(long numBytes) { ramAllocator.bytesUsed(numBytes); } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java index 9c7329f8ec4..0ece8dee33a 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java @@ -144,5 +144,4 @@ class DocumentsWriterRAMAllocator { String toMB(long v) { return nf.format(v/1024./1024.); } - } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java index f915cfbe322..e7019a3f165 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java @@ -121,8 +121,8 @@ abstract class DocumentsWriterThreadPool { throw new ThreadInterruptedException(ie); } - globalLock = true; pauseAllThreads(); + globalLock = true; } finally { lock.unlock(); } diff --git a/lucene/src/java/org/apache/lucene/index/FieldsWriter.java b/lucene/src/java/org/apache/lucene/index/FieldsWriter.java index 2b29270f73d..40c57afbcc0 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/FieldsWriter.java @@ -117,10 +117,9 @@ final class FieldsWriter // and adds a new entry for this document into the index // stream. This assumes the buffer was already written // in the correct fields format. - void flushDocument(int numStoredFields, RAMOutputStream buffer) throws IOException { + void startDocument(int numStoredFields) throws IOException { indexStream.writeLong(fieldsStream.getFilePointer()); fieldsStream.writeVInt(numStoredFields); - buffer.writeTo(fieldsStream); } void skipDocument() throws IOException { @@ -169,8 +168,8 @@ final class FieldsWriter } } - final void writeField(FieldInfo fi, Fieldable field) throws IOException { - fieldsStream.writeVInt(fi.number); + final void writeField(int fieldNumber, Fieldable field) throws IOException { + fieldsStream.writeVInt(fieldNumber); byte bits = 0; if (field.isTokenized()) bits |= FieldsWriter.FIELD_IS_TOKENIZED; @@ -226,7 +225,7 @@ final class FieldsWriter for (Fieldable field : fields) { if (field.isStored()) - writeField(fieldInfos.fieldInfo(field.name()), field); + writeField(fieldInfos.fieldInfo(field.name()).number, field); } } } diff --git a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java index dc59ab80423..8768b092208 100644 --- a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java @@ -20,21 +20,14 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; -import org.apache.lucene.index.DocumentsWriterPerThread.DocWriter; import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.PostingsConsumer; -import org.apache.lucene.index.codecs.TermsConsumer; import org.apache.lucene.util.BytesRef; final class FreqProxTermsWriter extends TermsHashConsumer { - @Override - void closeDocStore(SegmentWriteState state) {} - @Override void abort() {} @@ -112,8 +105,7 @@ final class FreqProxTermsWriter extends TermsHashConsumer { } @Override - DocWriter finishDocument() throws IOException { - return null; + void finishDocument(TermsHash termsHash) throws IOException { } @Override diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 2f53492e617..60ddde46f0e 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -413,7 +413,7 @@ public class IndexWriter implements Closeable { // this method is called: poolReaders = true; - flush(true, true, false); + flush(true, false); // Prevent segmentInfos from changing while opening the // reader; in theory we could do similar retry logic, @@ -1440,7 +1440,7 @@ public class IndexWriter implements Closeable { // Only allow a new merge to be triggered if we are // going to wait for merges: if (!hitOOM) { - flush(waitForMerges, true, true); + flush(waitForMerges, true); } if (waitForMerges) @@ -1961,7 +1961,7 @@ public class IndexWriter implements Closeable { if (infoStream != null) message("optimize: index now " + segString()); - flush(true, false, true); + flush(true, true); synchronized(this) { resetMergeExceptions(); @@ -2490,7 +2490,7 @@ public class IndexWriter implements Closeable { try { if (infoStream != null) message("flush at addIndexes(Directory...)"); - flush(true, false, true); + flush(true, true); int docCount = 0; List infos = new ArrayList(); @@ -2537,7 +2537,7 @@ public class IndexWriter implements Closeable { } // Update SI appropriately - info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile()); + info.setDocStoreSegment(newDsName); info.dir = directory; info.name = newSegName; @@ -2595,8 +2595,7 @@ public class IndexWriter implements Closeable { SegmentInfo info = null; synchronized(this) { - info = new SegmentInfo(mergedName, docCount, directory, false, -1, - null, false, merger.hasProx(), merger.getCodec()); + info = new SegmentInfo(mergedName, docCount, directory, false, merger.hasProx(), merger.getCodec()); setDiagnostics(info, "addIndexes(IndexReader...)"); segmentInfos.add(info); checkpoint(); @@ -2705,7 +2704,7 @@ public class IndexWriter implements Closeable { if (infoStream != null) message("prepareCommit: flush"); - flush(true, true, true); + flush(true, true); startCommit(0, commitUserData); } @@ -2826,18 +2825,18 @@ public class IndexWriter implements Closeable { * @param flushDeletes whether pending deletes should also * be flushed */ - protected final void flush(boolean triggerMerge, boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException { + protected final void flush(boolean triggerMerge, boolean flushDeletes) throws CorruptIndexException, IOException { // We can be called during close, when closing==true, so we must pass false to ensureOpen: ensureOpen(false); - if (doFlush(flushDocStores, flushDeletes) && triggerMerge) + if (doFlush(flushDeletes) && triggerMerge) maybeMerge(); } // TODO: this method should not have to be entirely // synchronized, ie, merges should be allowed to commit // even while a flush is happening - private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes) throws CorruptIndexException, IOException { - return docWriter.flushAllThreads(flushDocStores, flushDeletes); + private synchronized final boolean doFlush(boolean flushDeletes) throws CorruptIndexException, IOException { + return docWriter.flushAllThreads(flushDeletes); // nocommit // try { // try { @@ -2998,7 +2997,6 @@ public class IndexWriter implements Closeable { // nocommit //docWriter.remapDeletes(segmentInfos, merger.getDocMaps(), merger.getDelCounts(), merge, mergedDocCount); - setMergeDocStoreIsCompoundFile(merge); merge.info.setHasProx(merger.hasProx()); segmentInfos.subList(start, start + merge.segments.size()).clear(); @@ -3199,108 +3197,13 @@ public class IndexWriter implements Closeable { final SegmentInfos sourceSegments = merge.segments; final int end = sourceSegments.size(); - // Check whether this merge will allow us to skip - // merging the doc stores (stored field & vectors). - // This is a very substantial optimization (saves tons - // of IO). - - Directory lastDir = directory; - String lastDocStoreSegment = null; - int next = -1; - - boolean mergeDocStores = false; - boolean doFlushDocStore = false; - // nocommit - //final String currentDocStoreSegment = docWriter.getDocStoreSegment(); - - // Test each segment to be merged: check if we need to - // flush/merge doc stores - for (int i = 0; i < end; i++) { - SegmentInfo si = sourceSegments.info(i); - - // If it has deletions we must merge the doc stores - if (si.hasDeletions()) - mergeDocStores = true; - - // If it has its own (private) doc stores we must - // merge the doc stores - if (-1 == si.getDocStoreOffset()) - mergeDocStores = true; - - // If it has a different doc store segment than - // previous segments, we must merge the doc stores - String docStoreSegment = si.getDocStoreSegment(); - if (docStoreSegment == null) - mergeDocStores = true; - else if (lastDocStoreSegment == null) - lastDocStoreSegment = docStoreSegment; - else if (!lastDocStoreSegment.equals(docStoreSegment)) - mergeDocStores = true; - - // Segments' docScoreOffsets must be in-order, - // contiguous. For the default merge policy now - // this will always be the case but for an arbitrary - // merge policy this may not be the case - if (-1 == next) - next = si.getDocStoreOffset() + si.docCount; - else if (next != si.getDocStoreOffset()) - mergeDocStores = true; - else - next = si.getDocStoreOffset() + si.docCount; - - // If the segment comes from a different directory - // we must merge - if (lastDir != si.dir) - mergeDocStores = true; - - // If the segment is referencing the current "live" - // doc store outputs then we must merge - // nocommit -// if (si.getDocStoreOffset() != -1 && currentDocStoreSegment != null && si.getDocStoreSegment().equals(currentDocStoreSegment)) { -// doFlushDocStore = true; -// } - } - - final int docStoreOffset; - final String docStoreSegment; - final boolean docStoreIsCompoundFile; - - if (mergeDocStores) { - docStoreOffset = -1; - docStoreSegment = null; - docStoreIsCompoundFile = false; - } else { - SegmentInfo si = sourceSegments.info(0); - docStoreOffset = si.getDocStoreOffset(); - docStoreSegment = si.getDocStoreSegment(); - docStoreIsCompoundFile = si.getDocStoreIsCompoundFile(); - } - - if (mergeDocStores && doFlushDocStore) { - // SegmentMerger intends to merge the doc stores - // (stored fields, vectors), and at least one of the - // segments to be merged refers to the currently - // live doc stores. - - // TODO: if we know we are about to merge away these - // newly flushed doc store files then we should not - // make compound file out of them... - if (infoStream != null) - message("now flush at merge"); - doFlush(true, false); - } - merge.increfDone = true; - merge.mergeDocStores = mergeDocStores; - // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment // names. merge.info = new SegmentInfo(newSegmentName(), 0, - directory, false, docStoreOffset, - docStoreSegment, - docStoreIsCompoundFile, + directory, false, false, null); @@ -3308,7 +3211,6 @@ public class IndexWriter implements Closeable { Map details = new HashMap(); details.put("optimize", Boolean.toString(merge.optimize)); details.put("mergeFactor", Integer.toString(end)); - details.put("mergeDocStores", Boolean.toString(mergeDocStores)); setDiagnostics(merge.info, "merge", details); // Also enroll the merged segment into mergingSegments; @@ -3368,23 +3270,6 @@ public class IndexWriter implements Closeable { runningMerges.remove(merge); } - private synchronized void setMergeDocStoreIsCompoundFile(MergePolicy.OneMerge merge) { - final String mergeDocStoreSegment = merge.info.getDocStoreSegment(); - if (mergeDocStoreSegment != null && !merge.info.getDocStoreIsCompoundFile()) { - final int size = segmentInfos.size(); - for(int i=0;i dss = new HashSet(); - // This is try/finally to make sure merger's readers are // closed: boolean success = false; @@ -3426,7 +3307,7 @@ public class IndexWriter implements Closeable { // Hold onto the "live" reader; we will use this to // commit merged deletes - SegmentReader reader = merge.readers[i] = readerPool.get(info, merge.mergeDocStores, + SegmentReader reader = merge.readers[i] = readerPool.get(info, true, MERGE_READ_BUFFER_SIZE, -config.getReaderTermsIndexDivisor()); @@ -3436,14 +3317,6 @@ public class IndexWriter implements Closeable { SegmentReader clone = merge.readersClone[i] = (SegmentReader) reader.clone(true); merger.add(clone); - if (clone.hasDeletions()) { - mergeDocStores = true; - } - - if (info.getDocStoreOffset() != -1) { - dss.add(info.getDocStoreSegment()); - } - totDocCount += clone.numDocs(); } @@ -3453,40 +3326,12 @@ public class IndexWriter implements Closeable { merge.checkAborted(directory); - // If deletions have arrived and it has now become - // necessary to merge doc stores, go and open them: - if (mergeDocStores && !merge.mergeDocStores) { - merge.mergeDocStores = true; - synchronized(this) { - - // If 1) we must now merge doc stores, and 2) at - // least one of the segments we are merging uses - // the doc store we are now writing to, we must at - // this point force this doc store closed (by - // calling flush). If we didn't do this then the - // readers will attempt to open an IndexInput - // on files that have still-open IndexOutputs - // against them: - // nocommit -// if (dss.contains(docWriter.getDocStoreSegment())) { -// if (infoStream != null) -// message("now flush at mergeMiddle"); -// doFlush(true, false); -// } - } - - for(int i=0;i fieldsToFlush, SegmentWriteState state) throws IOException; - /** Close doc stores */ - abstract void closeDocStore(SegmentWriteState state) throws IOException; - abstract InvertedDocConsumerPerField addField(DocInverterPerField docInverterPerField, FieldInfo fieldInfo); abstract void startDocument() throws IOException; - abstract DocumentsWriterPerThread.DocWriter finishDocument() throws IOException; + abstract void finishDocument() throws IOException; /** Attempt to free RAM, returning true if any RAM was * freed */ diff --git a/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java b/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java index 762c8c484b0..189f97067ca 100644 --- a/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java @@ -22,7 +22,6 @@ import java.util.Map; abstract class InvertedDocEndConsumer { abstract void flush(Map fieldsToFlush, SegmentWriteState state) throws IOException; - abstract void closeDocStore(SegmentWriteState state) throws IOException; abstract void abort(); abstract void setFieldInfos(FieldInfos fieldInfos); abstract InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField, FieldInfo fieldInfo); diff --git a/lucene/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/src/java/org/apache/lucene/index/MergePolicy.java index afe1db22e83..8589a772697 100644 --- a/lucene/src/java/org/apache/lucene/index/MergePolicy.java +++ b/lucene/src/java/org/apache/lucene/index/MergePolicy.java @@ -67,7 +67,6 @@ public abstract class MergePolicy implements java.io.Closeable { public static class OneMerge { SegmentInfo info; // used by IndexWriter - boolean mergeDocStores; // used by IndexWriter boolean optimize; // used by IndexWriter boolean increfDone; // used by IndexWriter boolean registerDone; // used by IndexWriter @@ -157,9 +156,6 @@ public abstract class MergePolicy implements java.io.Closeable { b.append(" into ").append(info.name); if (optimize) b.append(" [optimize]"); - if (mergeDocStores) { - b.append(" [mergeDocStores]"); - } return b.toString(); } } diff --git a/lucene/src/java/org/apache/lucene/index/NormsWriter.java b/lucene/src/java/org/apache/lucene/index/NormsWriter.java index 209db92c103..31ca61b5259 100644 --- a/lucene/src/java/org/apache/lucene/index/NormsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/NormsWriter.java @@ -102,10 +102,6 @@ final class NormsWriter extends InvertedDocEndConsumer { } } - @Override - void closeDocStore(SegmentWriteState state) {} - - @Override void finishDocument() throws IOException {} diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java index abd4766aad6..d8d93379ebb 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java @@ -70,11 +70,11 @@ public final class SegmentInfo { long sizeInBytes = -1; // total byte size of all of our files (computed on demand) - private int docStoreOffset; // if this segment shares stored fields & vectors, this + @Deprecated private int docStoreOffset; // if this segment shares stored fields & vectors, this // offset is where in that file this segment's docs begin - private String docStoreSegment; // name used to derive fields/vectors file we share with + @Deprecated private String docStoreSegment; // name used to derive fields/vectors file we share with // other segments - private boolean docStoreIsCompoundFile; // whether doc store files are stored in compound file (*.cfx) + @Deprecated private boolean docStoreIsCompoundFile; // whether doc store files are stored in compound file (*.cfx) private int delCount; // How many deleted docs in this segment @@ -87,20 +87,23 @@ public final class SegmentInfo { private Map diagnostics; - public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, int docStoreOffset, - String docStoreSegment, boolean docStoreIsCompoundFile, boolean hasProx, Codec codec) { + public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasProx, Codec codec) { + this(name, docCount, dir, isCompoundFile, -1, null, false, hasProx, codec); + } + + private SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, int docStoreOffset, + String docStoreSegment, boolean docStoreIsCompoundFile, boolean hasProx, Codec codec) { this.name = name; this.docCount = docCount; this.dir = dir; delGen = NO; this.isCompoundFile = isCompoundFile; - this.docStoreOffset = docStoreOffset; - this.docStoreSegment = docStoreSegment; - this.docStoreIsCompoundFile = docStoreIsCompoundFile; this.hasProx = hasProx; this.codec = codec; delCount = 0; - assert docStoreOffset == -1 || docStoreSegment != null: "dso=" + docStoreOffset + " dss=" + docStoreSegment + " docCount=" + docCount; + this.docStoreOffset = docStoreOffset; + this.docStoreIsCompoundFile = docStoreIsCompoundFile; + this.docStoreSegment = docStoreSegment; } /** @@ -176,11 +179,13 @@ public final class SegmentInfo { docStoreSegment = name; docStoreIsCompoundFile = false; } + if (format > DefaultSegmentInfosWriter.FORMAT_4_0) { // pre-4.0 indexes write a byte if there is a single norms file byte b = input.readByte(); assert 1 == b; } + int numNormGen = input.readInt(); if (numNormGen == NO) { normGen = null; @@ -367,32 +372,24 @@ public final class SegmentInfo { assert delCount <= docCount; } + @Deprecated public int getDocStoreOffset() { return docStoreOffset; } + @Deprecated public boolean getDocStoreIsCompoundFile() { return docStoreIsCompoundFile; } - void setDocStoreIsCompoundFile(boolean v) { - docStoreIsCompoundFile = v; - clearFiles(); - } - + @Deprecated public String getDocStoreSegment() { return docStoreSegment; } - - void setDocStoreOffset(int offset) { - docStoreOffset = offset; - clearFiles(); - } - void setDocStore(int offset, String segment, boolean isCompoundFile) { - docStoreOffset = offset; - docStoreSegment = segment; - docStoreIsCompoundFile = isCompoundFile; + @Deprecated + public void setDocStoreSegment(String docStoreSegment) { + this.docStoreSegment = docStoreSegment; clearFiles(); } @@ -403,12 +400,14 @@ public final class SegmentInfo { output.writeString(name); output.writeInt(docCount); output.writeLong(delGen); + output.writeInt(docStoreOffset); if (docStoreOffset != -1) { output.writeString(docStoreSegment); output.writeByte((byte) (docStoreIsCompoundFile ? 1:0)); } + if (normGen == null) { output.writeInt(NO); } else { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java index 03426549f0e..8580d5f68fc 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java @@ -65,12 +65,6 @@ final class SegmentMerger { private final CheckAbort checkAbort; - // Whether we should merge doc stores (stored fields and - // vectors files). When all segments we are merging - // already share the same doc store files, we don't need - // to merge the doc stores. - private boolean mergeDocStores; - /** Maximum number of contiguous documents to bulk-copy when merging stored fields */ private final static int MAX_RAW_MERGE_DOCS = 4192; @@ -127,22 +121,6 @@ final class SegmentMerger { * @throws IOException if there is a low-level IO error */ final int merge() throws CorruptIndexException, IOException { - return merge(true); - } - - /** - * Merges the readers specified by the {@link #add} method - * into the directory passed to the constructor. - * @param mergeDocStores if false, we will not merge the - * stored fields nor vectors files - * @return The number of documents that were merged - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - final int merge(boolean mergeDocStores) throws CorruptIndexException, IOException { - - this.mergeDocStores = mergeDocStores; - // NOTE: it's important to add calls to // checkAbort.work(...) if you make any changes to this // method that will spend alot of time. The frequency @@ -154,7 +132,7 @@ final class SegmentMerger { mergeTerms(); mergeNorms(); - if (mergeDocStores && fieldInfos.hasVectors()) + if (fieldInfos.hasVectors()) mergeVectors(); return mergedDocs; @@ -179,9 +157,7 @@ final class SegmentMerger { // Basic files for (String ext : IndexFileNames.COMPOUND_EXTENSIONS_NOT_CODEC) { - if (mergeDocStores || (!ext.equals(IndexFileNames.FIELDS_EXTENSION) && - !ext.equals(IndexFileNames.FIELDS_INDEX_EXTENSION))) - fileSet.add(IndexFileNames.segmentFileName(segment, "", ext)); + fileSet.add(IndexFileNames.segmentFileName(segment, "", ext)); } codec.files(directory, info, fileSet); @@ -197,7 +173,7 @@ final class SegmentMerger { } // Vector files - if (fieldInfos.hasVectors() && mergeDocStores) { + if (fieldInfos.hasVectors()) { for (String ext : IndexFileNames.VECTOR_EXTENSIONS) { fileSet.add(IndexFileNames.segmentFileName(segment, "", ext)); } @@ -270,17 +246,20 @@ final class SegmentMerger { */ private final int mergeFields() throws CorruptIndexException, IOException { - if (!mergeDocStores) { - // When we are not merging by doc stores, their field - // name -> number mapping are the same. So, we start - // with the fieldInfos of the last segment in this - // case, to keep that numbering. - final SegmentReader sr = (SegmentReader) readers.get(readers.size()-1); - fieldInfos = (FieldInfos) sr.core.fieldInfos.clone(); - } else { - fieldInfos = new FieldInfos(); // merge field names - } + //nocommit +// if (!mergeDocStores) { +// // When we are not merging by doc stores, their field +// // name -> number mapping are the same. So, we start +// // with the fieldInfos of the last segment in this +// // case, to keep that numbering. +// final SegmentReader sr = (SegmentReader) readers.get(readers.size()-1); +// fieldInfos = (FieldInfos) sr.core.fieldInfos.clone(); +// } else { +// fieldInfos = new FieldInfos(); // merge field names +// } + fieldInfos = new FieldInfos(); // merge field names + for (IndexReader reader : readers) { if (reader instanceof SegmentReader) { SegmentReader segmentReader = (SegmentReader) reader; @@ -310,54 +289,45 @@ final class SegmentMerger { setMatchingSegmentReaders(); - if (mergeDocStores) { - // merge field values - final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos); + // merge field values + final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos); - try { - int idx = 0; - for (IndexReader reader : readers) { - final SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++]; - FieldsReader matchingFieldsReader = null; - if (matchingSegmentReader != null) { - final FieldsReader fieldsReader = matchingSegmentReader.getFieldsReader(); - if (fieldsReader != null && fieldsReader.canReadRawDocs()) { - matchingFieldsReader = fieldsReader; - } - } - if (reader.hasDeletions()) { - docCount += copyFieldsWithDeletions(fieldsWriter, - reader, matchingFieldsReader); - } else { - docCount += copyFieldsNoDeletions(fieldsWriter, - reader, matchingFieldsReader); + try { + int idx = 0; + for (IndexReader reader : readers) { + final SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++]; + FieldsReader matchingFieldsReader = null; + if (matchingSegmentReader != null) { + final FieldsReader fieldsReader = matchingSegmentReader.getFieldsReader(); + if (fieldsReader != null && fieldsReader.canReadRawDocs()) { + matchingFieldsReader = fieldsReader; } } - } finally { - fieldsWriter.close(); - } - - final String fileName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); - final long fdxFileLength = directory.fileLength(fileName); - - if (4+((long) docCount)*8 != fdxFileLength) - // This is most likely a bug in Sun JRE 1.6.0_04/_05; - // we detect that the bug has struck, here, and - // throw an exception to prevent the corruption from - // entering the index. See LUCENE-1282 for - // details. - throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - - } else { - // If we are skipping the doc stores, that means there - // are no deletions in any of these segments, so we - // just sum numDocs() of each segment to get total docCount - for (final IndexReader reader : readers) { - docCount += reader.numDocs(); + if (reader.hasDeletions()) { + docCount += copyFieldsWithDeletions(fieldsWriter, + reader, matchingFieldsReader); + } else { + docCount += copyFieldsNoDeletions(fieldsWriter, + reader, matchingFieldsReader); + } } + } finally { + fieldsWriter.close(); } - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, null, docCount, 0, termIndexInterval, codecs); + final String fileName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + final long fdxFileLength = directory.fileLength(fileName); + + if (4+((long) docCount)*8 != fdxFileLength) { + // This is most likely a bug in Sun JRE 1.6.0_04/_05; + // we detect that the bug has struck, here, and + // throw an exception to prevent the corruption from + // entering the index. See LUCENE-1282 for + // details. + throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); + } + + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecs); return docCount; } diff --git a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java index f94d6bc9bac..38e06dc35d8 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java @@ -33,9 +33,7 @@ public class SegmentWriteState { public final Directory directory; public final String segmentName; public final FieldInfos fieldInfos; - public final String docStoreSegmentName; public final int numDocs; - public int numDocsInStore; public final Collection flushedFiles; // Actual codec used @@ -61,16 +59,12 @@ public class SegmentWriteState { public final int maxSkipLevels = 10; public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, - String docStoreSegmentName, int numDocs, - int numDocsInStore, int termIndexInterval, - CodecProvider codecs) { + int numDocs, int termIndexInterval, CodecProvider codecs) { this.infoStream = infoStream; this.directory = directory; this.segmentName = segmentName; this.fieldInfos = fieldInfos; - this.docStoreSegmentName = docStoreSegmentName; this.numDocs = numDocs; - this.numDocsInStore = numDocsInStore; this.termIndexInterval = termIndexInterval; this.codec = codecs.getWriter(this); flushedFiles = new HashSet(); diff --git a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java index 0837fbae3b1..2dfc1176e9e 100644 --- a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java @@ -20,8 +20,6 @@ package org.apache.lucene.index; import java.io.IOException; import org.apache.lucene.document.Fieldable; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; @@ -29,39 +27,38 @@ import org.apache.lucene.util.RamUsageEstimator; final class StoredFieldsWriter { FieldsWriter fieldsWriter; - final FieldsWriter localFieldsWriter; final DocumentsWriterPerThread docWriter; final FieldInfos fieldInfos; int lastDocID; - private String docStoreSegment; + private String segment; - PerDoc[] docFreeList = new PerDoc[1]; int freeCount; - PerDoc doc; final DocumentsWriterPerThread.DocState docState; public StoredFieldsWriter(DocumentsWriterPerThread docWriter, FieldInfos fieldInfos) { this.docWriter = docWriter; this.fieldInfos = fieldInfos; this.docState = docWriter.docState; - localFieldsWriter = new FieldsWriter((IndexOutput) null, (IndexOutput) null, fieldInfos); + } + + private int numStoredFields; + private Fieldable[] storedFields; + private int[] fieldNumbers; + + public void reset() { + numStoredFields = 0; + storedFields = new Fieldable[1]; + fieldNumbers = new int[1]; } public void startDocument() { - if (doc != null) { - // Only happens if previous document hit non-aborting - // exception while writing stored fields into - // localFieldsWriter: - doc.reset(); - doc.docID = docState.docID; - } + reset(); } - - + public void flush(SegmentWriteState state) throws IOException { - if (state.numDocsInStore > 0) { + if (state.numDocs > 0) { // It's possible that all documents seen in this segment // hit non-aborting exceptions, in which case we will // not have yet init'd the FieldsWriter: @@ -69,86 +66,47 @@ final class StoredFieldsWriter { // Fill fdx file to include any final docs that we // skipped because they hit non-aborting exceptions - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); - } - - if (fieldsWriter != null) - fieldsWriter.flush(); - } - - private synchronized void initFieldsWriter() throws IOException { - if (fieldsWriter == null) { - docStoreSegment = docWriter.getDocStoreSegment(); - if (docStoreSegment != null) { - fieldsWriter = new FieldsWriter(docWriter.directory, - docStoreSegment, - fieldInfos); - docWriter.addOpenFile(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_EXTENSION)); - docWriter.addOpenFile(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); - lastDocID = 0; - } - } - } - - public void closeDocStore(SegmentWriteState state) throws IOException { - final int inc = state.numDocsInStore - lastDocID; - if (inc > 0) { - initFieldsWriter(); - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); + fill(state.numDocs); } if (fieldsWriter != null) { + fieldsWriter.flush(); fieldsWriter.close(); fieldsWriter = null; - assert docStoreSegment != null; - assert state.docStoreSegmentName != null; - assert docStoreSegment.equals(state.docStoreSegmentName): "fieldsWriter wrote to segment=" + docStoreSegment + " vs SegmentWriteState segment=" + state.docStoreSegmentName; lastDocID = 0; - String fieldsName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.FIELDS_EXTENSION); - String fieldsIdxName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + String fieldsName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.FIELDS_EXTENSION); + String fieldsIdxName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.FIELDS_INDEX_EXTENSION); state.flushedFiles.add(fieldsName); state.flushedFiles.add(fieldsIdxName); docWriter.removeOpenFile(fieldsName); docWriter.removeOpenFile(fieldsIdxName); - if (4+((long) state.numDocsInStore)*8 != state.directory.fileLength(fieldsIdxName)) - throw new RuntimeException("after flush: fdx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.fileLength(fieldsIdxName) + " length in bytes of " + fieldsIdxName + " file exists?=" + state.directory.fileExists(fieldsIdxName)); + if (4+((long) state.numDocs)*8 != state.directory.fileLength(fieldsIdxName)) { + throw new RuntimeException("after flush: fdx size mismatch: " + state.numDocs + " docs vs " + state.directory.fileLength(fieldsIdxName) + " length in bytes of " + fieldsIdxName + " file exists?=" + state.directory.fileExists(fieldsIdxName)); + } + } + + } + + private void initFieldsWriter() throws IOException { + if (fieldsWriter == null) { + segment = docWriter.getSegment(); + if (segment != null) { + fieldsWriter = new FieldsWriter(docWriter.directory, + segment, + fieldInfos); + docWriter.addOpenFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); + docWriter.addOpenFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); + lastDocID = 0; + } } } int allocCount; - PerDoc getPerDoc() { - if (freeCount == 0) { - allocCount++; - if (allocCount > docFreeList.length) { - // Grow our free list up front to make sure we have - // enough space to recycle all outstanding PerDoc - // instances - assert allocCount == 1+docFreeList.length; - docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - } - return new PerDoc(); - } else - return docFreeList[--freeCount]; - } - - public DocumentsWriterPerThread.DocWriter finishDocument() { - // If there were any stored fields in this doc, doc will - // be non-null; else it's null. - try { - return doc; - } finally { - doc = null; - } - } - void abort() { - if (doc != null) { - doc.abort(); - doc = null; - } + reset(); if (fieldsWriter != null) { try { @@ -162,83 +120,50 @@ final class StoredFieldsWriter { /** Fills in any hole in the docIDs */ void fill(int docID) throws IOException { - final int docStoreOffset = docWriter.getDocStoreOffset(); - // We must "catch up" for all docs before us // that had no stored fields: - final int end = docID+docStoreOffset; + final int end = docID; while(lastDocID < end) { fieldsWriter.skipDocument(); lastDocID++; } } - void finishDocument(PerDoc perDoc) throws IOException { + void finishDocument() throws IOException { assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument start"); + initFieldsWriter(); - - fill(perDoc.docID); - - // Append stored fields to the real FieldsWriter: - fieldsWriter.flushDocument(perDoc.numStoredFields, perDoc.fdt); - lastDocID++; - perDoc.reset(); - free(perDoc); + fill(docState.docID); + + if (fieldsWriter != null && numStoredFields > 0) { + fieldsWriter.startDocument(numStoredFields); + for (int i = 0; i < numStoredFields; i++) { + fieldsWriter.writeField(fieldNumbers[i], storedFields[i]); + } + lastDocID++; + } + + reset(); assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end"); } - public void addField(Fieldable field, FieldInfo fieldInfo) throws IOException { - if (doc == null) { - doc = getPerDoc(); - doc.docID = docState.docID; - localFieldsWriter.setFieldsStream(doc.fdt); - assert doc.numStoredFields == 0: "doc.numStoredFields=" + doc.numStoredFields; - assert 0 == doc.fdt.length(); - assert 0 == doc.fdt.getFilePointer(); - } - - localFieldsWriter.writeField(fieldInfo, field); - assert docState.testPoint("StoredFieldsWriterPerThread.processFields.writeField"); - doc.numStoredFields++; - } - public boolean freeRAM() { - return false; - } - - void free(PerDoc perDoc) { - assert freeCount < docFreeList.length; - assert 0 == perDoc.numStoredFields; - assert 0 == perDoc.fdt.length(); - assert 0 == perDoc.fdt.getFilePointer(); - docFreeList[freeCount++] = perDoc; - } - - class PerDoc extends DocumentsWriterPerThread.DocWriter { - final DocumentsWriterPerThread.PerDocBuffer buffer = docWriter.newPerDocBuffer(); - RAMOutputStream fdt = new RAMOutputStream(buffer); - int numStoredFields; - - void reset() { - fdt.reset(); - buffer.recycle(); - numStoredFields = 0; + public void addField(Fieldable field, FieldInfo fieldInfo) throws IOException { + if (numStoredFields == storedFields.length) { + int newSize = ArrayUtil.oversize(numStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJ_REF); + Fieldable[] newArray = new Fieldable[newSize]; + System.arraycopy(storedFields, 0, newArray, 0, numStoredFields); + storedFields = newArray; } - - @Override - public void abort() { - reset(); - free(this); - } - - @Override - public long sizeInBytes() { - return buffer.getSizeInBytes(); - } - - @Override - public void finish() throws IOException { - finishDocument(this); + + if (numStoredFields == fieldNumbers.length) { + fieldNumbers = ArrayUtil.grow(fieldNumbers); } + + storedFields[numStoredFields] = field; + fieldNumbers[numStoredFields] = fieldInfo.number; + numStoredFields++; + + assert docState.testPoint("StoredFieldsWriterPerThread.processFields.writeField"); } } diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java index 1fad12239e6..6fadbf36a44 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java @@ -20,9 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Map; -import org.apache.lucene.index.DocumentsWriterPerThread.DocWriter; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; @@ -31,7 +29,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { final DocumentsWriterPerThread docWriter; TermVectorsWriter termVectorsWriter; - PerDoc[] docFreeList = new PerDoc[1]; int freeCount; IndexOutput tvx; IndexOutput tvd; @@ -40,7 +37,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { final DocumentsWriterPerThread.DocState docState; final BytesRef flushTerm = new BytesRef(); - TermVectorsTermsWriter.PerDoc doc; // Used by perField when serializing the term vectors final ByteSliceReader vectorSliceReader = new ByteSliceReader(); @@ -55,40 +51,26 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { if (tvx != null) { - if (state.numDocsInStore > 0) + if (state.numDocs > 0) { // In case there are some final documents that we // didn't see (because they hit a non-aborting exception): - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); + fill(state.numDocs); + } tvx.flush(); tvd.flush(); tvf.flush(); - } - - for (final TermsHashConsumerPerField field : fieldsToFlush.values() ) { - TermVectorsTermsWriterPerField perField = (TermVectorsTermsWriterPerField) field; - perField.termsHashPerField.reset(); - perField.shrinkHash(); - } - } - - @Override - void closeDocStore(final SegmentWriteState state) throws IOException { - if (tvx != null) { - // At least one doc in this run had term vectors - // enabled - fill(state.numDocsInStore - docWriter.getDocStoreOffset()); + tvx.close(); tvf.close(); tvd.close(); tvx = null; - assert state.docStoreSegmentName != null; - String idxName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.VECTORS_INDEX_EXTENSION); - if (4+((long) state.numDocsInStore)*16 != state.directory.fileLength(idxName)) - throw new RuntimeException("after flush: tvx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.fileLength(idxName) + " length in bytes of " + idxName + " file exists?=" + state.directory.fileExists(idxName)); + String idxName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.VECTORS_INDEX_EXTENSION); + if (4+((long) state.numDocs)*16 != state.directory.fileLength(idxName)) + throw new RuntimeException("after flush: tvx size mismatch: " + state.numDocs + " docs vs " + state.directory.fileLength(idxName) + " length in bytes of " + idxName + " file exists?=" + state.directory.fileExists(idxName)); - String fldName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); - String docName = IndexFileNames.segmentFileName(state.docStoreSegmentName, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); + String fldName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); + String docName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); state.flushedFiles.add(idxName); state.flushedFiles.add(fldName); state.flushedFiles.add(docName); @@ -98,31 +80,20 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { docWriter.removeOpenFile(docName); lastDocID = 0; - } - } - int allocCount; + } - PerDoc getPerDoc() { - if (freeCount == 0) { - allocCount++; - if (allocCount > docFreeList.length) { - // Grow our free list up front to make sure we have - // enough space to recycle all outstanding PerDoc - // instances - assert allocCount == 1+docFreeList.length; - docFreeList = new PerDoc[ArrayUtil.oversize(allocCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; - } - return new PerDoc(); - } else - return docFreeList[--freeCount]; + for (final TermsHashConsumerPerField field : fieldsToFlush.values() ) { + TermVectorsTermsWriterPerField perField = (TermVectorsTermsWriterPerField) field; + perField.termsHashPerField.reset(); + perField.shrinkHash(); + } } /** Fills in no-term-vectors for all docs we haven't seen * since the last doc that had term vectors. */ void fill(int docID) throws IOException { - final int docStoreOffset = docWriter.getDocStoreOffset(); - final int end = docID+docStoreOffset; + final int end = docID; if (lastDocID < end) { final long tvfPosition = tvf.getFilePointer(); while(lastDocID < end) { @@ -137,18 +108,18 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { void initTermVectorsWriter() throws IOException { if (tvx == null) { - final String docStoreSegment = docWriter.getDocStoreSegment(); + final String segment = docWriter.getSegment(); - if (docStoreSegment == null) + if (segment == null) return; // If we hit an exception while init'ing the term // vector output files, we must abort this segment // because those files will be in an unknown // state: - String idxName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_INDEX_EXTENSION); - String docName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); - String fldName = IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); + String idxName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION); + String docName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); + String fldName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); tvx = docWriter.directory.createOutput(idxName); tvd = docWriter.directory.createOutput(docName); tvf = docWriter.directory.createOutput(fldName); @@ -165,53 +136,44 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { } } - void finishDocument(PerDoc perDoc) throws IOException { + @Override + void finishDocument(TermsHash termsHash) throws IOException { assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument start"); initTermVectorsWriter(); - fill(perDoc.docID); - + fill(docState.docID); + // Append term vectors to the real outputs: tvx.writeLong(tvd.getFilePointer()); tvx.writeLong(tvf.getFilePointer()); - tvd.writeVInt(perDoc.numVectorFields); - if (perDoc.numVectorFields > 0) { - for(int i=0;i 0) { + for(int i=0;i= 0; - if (!doVectors || numPostings == 0) - return; - if (numPostings > maxNumPostings) maxNumPostings = numPostings; - final IndexOutput tvf = termsWriter.doc.perDocTvf; - // This is called once, after inverting all occurrences // of a given field in the doc. At this point we flush // our hash into the DocWriter. @@ -123,8 +112,8 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { assert fieldInfo.storeTermVector; assert termsWriter.vectorFieldsInOrder(fieldInfo); - termsWriter.doc.addField(termsHashPerField.fieldInfo.number); TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray; + final IndexOutput tvf = termsWriter.tvf; // TODO: we may want to make this sort in same order // as Codec's terms dict? @@ -186,7 +175,6 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { } termsHashPerField.reset(); - termsHashPerField.termsHash.reset(); } void shrinkHash() { diff --git a/lucene/src/java/org/apache/lucene/index/TermsHash.java b/lucene/src/java/org/apache/lucene/index/TermsHash.java index 36e70bf33d0..09ce7e64b1f 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsHash.java +++ b/lucene/src/java/org/apache/lucene/index/TermsHash.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import org.apache.lucene.index.DocumentsWriterPerThread.DocWriter; import org.apache.lucene.util.BytesRef; /** This class implements {@link InvertedDocConsumer}, which @@ -99,14 +98,6 @@ final class TermsHash extends InvertedDocConsumer { } } - - @Override - void closeDocStore(SegmentWriteState state) throws IOException { - consumer.closeDocStore(state); - if (nextTermsHash != null) - nextTermsHash.closeDocStore(state); - } - @Override void flush(Map fieldsToFlush, final SegmentWriteState state) throws IOException { Map childFields = new HashMap(); @@ -144,20 +135,13 @@ final class TermsHash extends InvertedDocConsumer { } @Override - DocWriter finishDocument() throws IOException { - final DocumentsWriterPerThread.DocWriter doc = consumer.finishDocument(); - - final DocumentsWriterPerThread.DocWriter doc2; - if (nextTermsHash != null) { - doc2 = nextTermsHash.consumer.finishDocument(); - } else { - doc2 = null; - } - if (doc == null) { - return doc2; - } else { - doc.setNext(doc2); - return doc; + void finishDocument() throws IOException { + try { + consumer.finishDocument(this); + } finally { + if (nextTermsHash != null) { + nextTermsHash.consumer.finishDocument(nextTermsHash); + } } } diff --git a/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java b/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java index f76def1e44d..99b2de8e70f 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java @@ -23,10 +23,8 @@ import java.util.Map; abstract class TermsHashConsumer { abstract void flush(Map fieldsToFlush, final SegmentWriteState state) throws IOException; abstract void abort(); - abstract void closeDocStore(SegmentWriteState state) throws IOException; - abstract void startDocument() throws IOException; - abstract DocumentsWriterPerThread.DocWriter finishDocument() throws IOException; + abstract void finishDocument(TermsHash termsHash) throws IOException; abstract public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo); FieldInfos fieldInfos; diff --git a/lucene/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/src/test/org/apache/lucene/index/TestCodecs.java index 11921e3aa03..190ff1cd8df 100644 --- a/lucene/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/src/test/org/apache/lucene/index/TestCodecs.java @@ -281,7 +281,7 @@ public class TestCodecs extends MultiCodecTestCase { final Directory dir = new MockRAMDirectory(); this.write(fieldInfos, dir, fields); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, CodecProvider.getDefault().getWriter(null)); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, CodecProvider.getDefault().getWriter(null)); si.setHasProx(false); final FieldsProducer reader = si.getCodec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); @@ -319,7 +319,7 @@ public class TestCodecs extends MultiCodecTestCase { final Directory dir = new MockRAMDirectory(); this.write(fieldInfos, dir, fields); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, CodecProvider.getDefault().getWriter(null)); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, CodecProvider.getDefault().getWriter(null)); final FieldsProducer terms = si.getCodec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); @@ -602,7 +602,7 @@ public class TestCodecs extends MultiCodecTestCase { final int termIndexInterval = this.nextInt(13, 27); - final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, + final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, 10000, termIndexInterval, CodecProvider.getDefault()); final FieldsConsumer consumer = state.codec.fieldsConsumer(state); diff --git a/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java b/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java index 275adf12bc9..f354c284671 100644 --- a/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java +++ b/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java @@ -84,7 +84,7 @@ public class TestConcurrentMergeScheduler extends LuceneTestCase { writer.addDocument(doc); failure.setDoFail(); try { - writer.flush(true, false, true); + writer.flush(true, true); if (failure.hitExc) { fail("failed to hit IOException"); } diff --git a/lucene/src/test/org/apache/lucene/index/TestDoc.java b/lucene/src/test/org/apache/lucene/index/TestDoc.java index 779d74b21ba..8d544fc2902 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDoc.java +++ b/lucene/src/test/org/apache/lucene/index/TestDoc.java @@ -194,7 +194,7 @@ public class TestDoc extends LuceneTestCase { merger.closeReaders(); final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir, - useCompoundFile, -1, null, false, merger.hasProx(), merger.getCodec()); + useCompoundFile, merger.hasProx(), merger.getCodec()); if (useCompoundFile) { List filesToDelete = merger.createCompoundFile(merged + ".cfs", info); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index fc03329cdaf..3ba715420d2 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -1437,7 +1437,7 @@ public class TestIndexWriter extends LuceneTestCase { doc.add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for(int i=0;i<19;i++) writer.addDocument(doc); - writer.flush(false, true, true); + writer.flush(false, true); writer.close(); SegmentInfos sis = new SegmentInfos(); sis.read(dir); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java index e49d5c0f05d..bc4f7f6763d 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java @@ -146,7 +146,7 @@ public class TestIndexWriterReader extends LuceneTestCase { writer.setInfoStream(infoStream); // create the index createIndexNoClose(!optimize, "index1", writer); - writer.flush(false, true, true); + writer.flush(false, true); // create a 2nd index Directory dir2 = new MockRAMDirectory(); @@ -224,7 +224,7 @@ public class TestIndexWriterReader extends LuceneTestCase { writer.setInfoStream(infoStream); // create the index createIndexNoClose(!optimize, "index1", writer); - writer.flush(false, true, true); + writer.flush(false, true); // get a reader IndexReader r1 = writer.getReader(); @@ -426,7 +426,7 @@ public class TestIndexWriterReader extends LuceneTestCase { IndexReader r1 = writer.getReader(); assertEquals(0, r1.maxDoc()); createIndexNoClose(false, "index1", writer); - writer.flush(!optimize, true, true); + writer.flush(!optimize, true); IndexReader iwr1 = writer.getReader(); assertEquals(100, iwr1.maxDoc()); @@ -438,7 +438,7 @@ public class TestIndexWriterReader extends LuceneTestCase { Document d = createDocument(x, "index1", 5); writer.addDocument(d); } - writer.flush(false, true, true); + writer.flush(false, true); // verify the reader was reopened internally IndexReader iwr2 = writer.getReader(); assertTrue(iwr2 != r1); diff --git a/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java index 29c9d611c5b..36abeee5c15 100644 --- a/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java +++ b/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java @@ -73,8 +73,8 @@ public class TestSegmentMerger extends LuceneTestCase { merger.closeReaders(); assertTrue(docsMerged == 2); //Should be able to open a new SegmentReader against the new directory - SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, -1, - null, false, merger.hasProx(), merger.getCodec()), BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null); + SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, + merger.hasProx(), merger.getCodec()), BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null); assertTrue(mergedReader != null); assertTrue(mergedReader.numDocs() == 2); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java b/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java index c50478b3664..afb8f2b8a85 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java @@ -134,7 +134,7 @@ public class TestSurrogates extends LuceneTestCaseJ4 { dir.createOutput(segName + ".frq").close(); // !!hack alert!! stuffing uniqueTermCount in as docCount - return new SegmentInfo(segName, uniqueTermCount, dir, false, -1, null, false, true, codec); + return new SegmentInfo(segName, uniqueTermCount, dir, false, true, codec); } private String toHexString(Term t) { From 8024f7f00a00fe4cac78f3f325532daf47cc364b Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Tue, 27 Jul 2010 20:33:05 +0000 Subject: [PATCH 006/200] LUCENE-2561: Fix most of the thread-safety and exception handling problems in the realtime search branch git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@979856 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 261 +++++++------- .../index/DocumentsWriterPerThread.java | 49 ++- .../index/DocumentsWriterThreadPool.java | 84 ++++- .../apache/lucene/index/IndexFileDeleter.java | 17 +- .../apache/lucene/index/IndexReaderPool.java | 263 ++++++++++++++ .../org/apache/lucene/index/IndexWriter.java | 323 ++++-------------- .../lucene/index/StoredFieldsWriter.java | 5 - .../lucene/index/TermVectorsTermsWriter.java | 15 +- .../org/apache/lucene/index/TermsHash.java | 9 +- .../apache/lucene/store/FilterDirectory.java | 111 ++++++ .../index/TestConcurrentMergeScheduler.java | 2 +- .../apache/lucene/index/TestIndexWriter.java | 13 +- .../index/TestIndexWriterExceptions.java | 3 +- .../index/TestNRTReaderWithThreads.java | 3 +- .../lucene/index/TestThreadedOptimize.java | 3 +- 15 files changed, 697 insertions(+), 464 deletions(-) create mode 100644 lucene/src/java/org/apache/lucene/index/IndexReaderPool.java create mode 100644 lucene/src/java/org/apache/lucene/store/FilterDirectory.java diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index f368854ed50..c1e3e8e4d0b 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -2,13 +2,13 @@ package org.apache.lucene.index; import java.io.IOException; import java.io.PrintStream; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; @@ -24,6 +24,8 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; /** @@ -51,7 +53,7 @@ final class DocumentsWriter { private final DocumentsWriterThreadPool threadPool; private final Lock sequenceIDLock = new ReentrantLock(); - private final Directory directory; + private final Directory openFilesTrackingDirectory; final IndexWriter indexWriter; final IndexWriterConfig config; @@ -69,7 +71,14 @@ final class DocumentsWriter { private Map minSequenceIDsPerThread = new HashMap(); public DocumentsWriter(Directory directory, IndexWriter indexWriter, IndexWriterConfig config) { - this.directory = directory; + this.openFilesTrackingDirectory = new FilterDirectory(directory) { + @Override public IndexOutput createOutput(final String name) throws IOException { + addOpenFile(name); + return super.createOutput(name); + } + }; + + //this.openFilesTrackingDirectory = directory; this.indexWriter = indexWriter; this.config = config; this.maxBufferedDocs = config.getMaxBufferedDocs(); @@ -111,7 +120,7 @@ final class DocumentsWriter { } DocumentsWriterPerThread newDocumentsWriterPerThread() { - DocumentsWriterPerThread perThread = new DocumentsWriterPerThread(directory, this, config + DocumentsWriterPerThread perThread = new DocumentsWriterPerThread(openFilesTrackingDirectory, this, config .getIndexingChain()); sequenceIDLock.lock(); try { @@ -127,13 +136,23 @@ final class DocumentsWriter { return updateDocument(null, doc, analyzer); } + private final static class UpdateResult { + long sequenceID; + boolean flushed; + + UpdateResult(long sequenceID) { + this.sequenceID = sequenceID; + flushed = false; + } + } + long updateDocument(final Term delTerm, final Document doc, final Analyzer analyzer) throws CorruptIndexException, IOException { - long seqID = threadPool.executePerThread(this, doc, - new DocumentsWriterThreadPool.PerThreadTask() { + UpdateResult result = threadPool.executePerThread(this, doc, + new DocumentsWriterThreadPool.PerThreadTask() { @Override - public Long process(final DocumentsWriterPerThread perThread) throws IOException { + public UpdateResult process(final DocumentsWriterPerThread perThread) throws IOException { long perThreadRAMUsedBeforeAdd = perThread.numBytesUsed; perThread.addDocument(doc, analyzer); @@ -154,16 +173,23 @@ final class DocumentsWriter { sequenceIDLock.unlock(); } + UpdateResult result = new UpdateResult(sequenceID); if (finishAddDocument(perThread, perThreadRAMUsedBeforeAdd)) { + result.flushed = true; super.clearThreadBindings(); } - return sequenceID; + return result; } }); + + if (result == null) { + return -1; + } - indexWriter.maybeMerge(); - - return seqID; + if (result.flushed) { + indexWriter.maybeMerge(); + } + return result.sequenceID; } private final boolean finishAddDocument(DocumentsWriterPerThread perThread, @@ -257,17 +283,12 @@ final class DocumentsWriter { final boolean flushAllThreads(final boolean flushDeletes) throws IOException { - return threadPool.executeAllThreads(new DocumentsWriterThreadPool.AllThreadsTask() { + + return threadPool.executeAllThreads(this, new DocumentsWriterThreadPool.AllThreadsTask() { @Override public Boolean process(Iterator threadsIterator) throws IOException { boolean anythingFlushed = false; - if (flushDeletes) { - if (applyDeletes(indexWriter.segmentInfos)) { - indexWriter.checkpoint(); - } - } - while (threadsIterator.hasNext()) { DocumentsWriterPerThread perThread = threadsIterator.next(); final int numDocs = perThread.getNumDocsInRAM(); @@ -282,6 +303,7 @@ final class DocumentsWriter { if (flushDocs) { SegmentInfo newSegment = perThread.flush(); + newSegment.dir = indexWriter.getDirectory(); if (newSegment != null) { anythingFlushed = true; @@ -315,10 +337,8 @@ final class DocumentsWriter { } /** Build compound file for the segment we just flushed */ - void createCompoundFile(String segment, DocumentsWriterPerThread perThread) throws IOException { - - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, - IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); + void createCompoundFile(String compoundFileName, DocumentsWriterPerThread perThread) throws IOException { + CompoundFileWriter cfsWriter = new CompoundFileWriter(openFilesTrackingDirectory, compoundFileName); for(String fileName : perThread.flushState.flushedFiles) { cfsWriter.addFile(fileName); } @@ -327,49 +347,53 @@ final class DocumentsWriter { cfsWriter.close(); } - // nocommit void finishFlushedSegment(SegmentInfo newSegment, DocumentsWriterPerThread perThread) throws IOException { - synchronized(indexWriter) { - indexWriter.segmentInfos.add(newSegment); - indexWriter.checkpoint(); + SegmentReader reader = indexWriter.readerPool.get(newSegment, false); + try { + applyDeletes(reader, newSegment.getMinSequenceID(), newSegment.getMaxSequenceID(), perThread.sequenceIDs); + } finally { + indexWriter.readerPool.release(reader); + } - SegmentReader reader = indexWriter.readerPool.get(newSegment, false); - boolean any = false; + if (indexWriter.useCompoundFile(newSegment)) { + String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); + message("creating compound file " + compoundFileName); + // Now build compound file + boolean success = false; try { - any = applyDeletes(reader, newSegment.getMinSequenceID(), newSegment.getMaxSequenceID(), perThread.sequenceIDs); + createCompoundFile(compoundFileName, perThread); + success = true; } finally { - indexWriter.readerPool.release(reader); - } - if (any) { - indexWriter.checkpoint(); - } - - if (indexWriter.mergePolicy.useCompoundFile(indexWriter.segmentInfos, newSegment)) { - // Now build compound file - boolean success = false; - try { - createCompoundFile(newSegment.name, perThread); - success = true; - } finally { - if (!success) { - if (infoStream != null) { - message("hit exception " + - "reating compound file for newly flushed segment " + newSegment.name); - } - indexWriter.deleter.deleteFile(IndexFileNames.segmentFileName(newSegment.name, "", - IndexFileNames.COMPOUND_FILE_EXTENSION)); + if (!success) { + if (infoStream != null) { + message("hit exception " + + "reating compound file for newly flushed segment " + newSegment.name); } + indexWriter.deleter.deleteFile(IndexFileNames.segmentFileName(newSegment.name, "", + IndexFileNames.COMPOUND_FILE_EXTENSION)); + for (String file : perThread.flushState.flushedFiles) { + indexWriter.deleter.deleteFile(file); + } + } - - synchronized(indexWriter) { - newSegment.setUseCompoundFile(true); - indexWriter.checkpoint(); - // In case the files we just merged into a CFS were - // not previously checkpointed: - indexWriter.deleter.deleteNewFiles(perThread.closedFiles()); - } + } + + for (String file : perThread.flushState.flushedFiles) { + indexWriter.deleter.deleteFile(file); + } + + newSegment.setUseCompoundFile(true); + + synchronized(openFiles) { + openFiles.remove(compoundFileName); } } + + synchronized(openFiles) { + openFiles.removeAll(perThread.flushState.flushedFiles); + } + + indexWriter.addNewSegment(newSegment); } // Returns true if an abort is in progress @@ -400,6 +424,7 @@ final class DocumentsWriter { if (perThread.getNumDocsInRAM() == maxBufferedDocs) { flushSegment(perThread); assert perThread.getNumDocsInRAM() == 0; + return true; } @@ -413,48 +438,57 @@ final class DocumentsWriter { } SegmentInfo newSegment = perThread.flush(); + newSegment.dir = indexWriter.getDirectory(); - if (newSegment != null) { - finishFlushedSegment(newSegment, perThread); - return true; - } - return false; + finishFlushedSegment(newSegment, perThread); + return true; } void abort() throws IOException { - threadPool.abort(); - try { - try { - abortedFiles = openFiles(); - } catch (Throwable t) { - abortedFiles = null; + threadPool.abort(new DocumentsWriterThreadPool.AbortTask() { + + @Override + void abort() throws IOException { + try { + abortedFiles = openFiles(); + } catch (Throwable t) { + abortedFiles = null; + } + + deletesInRAM.clear(); + // nocommit + // deletesFlushed.clear(); + + openFiles.clear(); + deletesInRAM.clear(); } - - deletesInRAM.clear(); - // nocommit - // deletesFlushed.clear(); - - openFiles.clear(); - } finally { - threadPool.finishAbort(); - } - + }); } - final List openFiles = new ArrayList(); + final Set openFiles = new HashSet(); private Collection abortedFiles; // List of files that were written before last abort() - /* * Returns Collection of files in use by this instance, * including any flushed segments. */ @SuppressWarnings("unchecked") - List openFiles() { + private Collection openFiles() { synchronized(openFiles) { - return (List) ((ArrayList) openFiles).clone(); + return (Set) ((HashSet) openFiles).clone(); } } + void addOpenFile(String file) { + synchronized(openFiles) { + openFiles.add(file); + } + } + + void removeOpenFile(String file) { + synchronized(openFiles) { + openFiles.remove(file); + } + } Collection abortedFiles() { return abortedFiles; @@ -480,27 +514,29 @@ final class DocumentsWriter { // } boolean applyDeletes(SegmentInfos infos) throws IOException { - synchronized(indexWriter) { - if (!hasDeletes()) - return false; - - final long t0 = System.currentTimeMillis(); - - if (infoStream != null) { - message("apply " + deletesInRAM.getNumDeletes() + " buffered deletes on " + - +infos.size() + " segments."); - } - - final int infosEnd = infos.size(); - - boolean any = false; - for (int i = 0; i < infosEnd; i++) { - - // Make sure we never attempt to apply deletes to - // segment in external dir - assert infos.info(i).dir == directory; - - SegmentInfo si = infos.info(i); + if (!hasDeletes()) + return false; + + final long t0 = System.currentTimeMillis(); + + if (infoStream != null) { + message("apply " + deletesInRAM.getNumDeletes() + " buffered deletes on " + + +infos.size() + " segments."); + } + + final int infosEnd = infos.size(); + + boolean any = false; + for (int i = 0; i < infosEnd; i++) { + + // Make sure we never attempt to apply deletes to + // segment in external dir + assert infos.info(i).dir == indexWriter.getDirectory(); + + SegmentInfo si = infos.info(i); + // we have to synchronize here, because we need a write lock on + // the segment in order to apply deletes + synchronized (indexWriter) { SegmentReader reader = indexWriter.readerPool.get(si, false); try { any |= applyDeletes(reader, si.getMinSequenceID(), si.getMaxSequenceID(), null); @@ -508,13 +544,13 @@ final class DocumentsWriter { indexWriter.readerPool.release(reader); } } - - if (infoStream != null) { - message("apply deletes took " + (System.currentTimeMillis() - t0) + " msec"); - } - - return any; } + + if (infoStream != null) { + message("apply deletes took " + (System.currentTimeMillis() - t0) + " msec"); + } + + return any; } // Apply buffered delete terms, queries and docIDs to the @@ -642,9 +678,6 @@ final class DocumentsWriter { } void message(String message) { - if (infoStream != null) { - indexWriter.message("DW: " + message); - } + indexWriter.message("DW: " + message); } - } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 7233463f7de..36b5707196b 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -1,15 +1,32 @@ package org.apache.lucene.index; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.io.IOException; import java.io.PrintStream; -import java.util.ArrayList; -import java.util.List; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; public class DocumentsWriterPerThread { @@ -84,6 +101,7 @@ public class DocumentsWriterPerThread { * currently buffered docs. This resets our state, * discarding any docs added since last flush. */ void abort() throws IOException { + assert aborting; try { if (infoStream != null) { message("docWriter: now abort"); @@ -124,11 +142,11 @@ public class DocumentsWriterPerThread { SegmentWriteState flushState; long[] sequenceIDs = new long[8]; - final List closedFiles = new ArrayList(); long numBytesUsed; public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent, IndexingChain indexingChain) { + parent.indexWriter.testPoint("DocumentsWriterPerThread.init start"); this.directory = directory; this.parent = parent; this.writer = parent.indexWriter; @@ -141,7 +159,6 @@ public class DocumentsWriterPerThread { if (consumer instanceof DocFieldProcessor) { docFieldProcessor = (DocFieldProcessor) consumer; } - } void setAborting() { @@ -288,33 +305,11 @@ public class DocumentsWriterPerThread { return segment; } - @SuppressWarnings("unchecked") - List closedFiles() { - return (List) ((ArrayList) closedFiles).clone(); - } - - - void addOpenFile(String name) { - synchronized(parent.openFiles) { - assert !parent.openFiles.contains(name); - parent.openFiles.add(name); - } - } - - void removeOpenFile(String name) { - synchronized(parent.openFiles) { - assert parent.openFiles.contains(name); - parent.openFiles.remove(name); - } - closedFiles.add(name); - } - void bytesUsed(long numBytes) { ramAllocator.bytesUsed(numBytes); } void message(String message) { - if (infoStream != null) - writer.message("DW: " + message); + writer.message("DW: " + message); } } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java index e7019a3f165..ec7956895ef 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java @@ -1,5 +1,22 @@ package org.apache.lucene.index; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import java.io.IOException; import java.util.Iterator; import java.util.concurrent.locks.Condition; @@ -30,6 +47,10 @@ abstract class DocumentsWriterThreadPool { abstract T process(final Iterator threadsIterator) throws IOException; } + public static abstract class AbortTask { + abstract void abort() throws IOException; + } + protected abstract static class ThreadState { private DocumentsWriterPerThread perThread; private boolean isIdle = true; @@ -95,11 +116,29 @@ abstract class DocumentsWriterThreadPool { return true; } - void abort() throws IOException { - pauseAllThreads(); - aborting = true; - for (ThreadState state : allThreadStates) { - state.perThread.abort(); + void abort(AbortTask task) throws IOException { + lock.lock(); + try { + if (!aborting) { + aborting = true; + pauseAllThreads(); + for (ThreadState state : allThreadStates) { + state.perThread.aborting = true; + } + + try { + for (ThreadState state : allThreadStates) { + state.perThread.abort(); + } + + task.abort(); + } finally { + aborting = false; + resumeAllThreads(); + } + } + } finally { + lock.unlock(); } } @@ -108,7 +147,7 @@ abstract class DocumentsWriterThreadPool { resumeAllThreads(); } - public T executeAllThreads(AllThreadsTask task) throws IOException { + public T executeAllThreads(DocumentsWriter documentsWriter, AllThreadsTask task) throws IOException { T result = null; lock.lock(); @@ -120,19 +159,21 @@ abstract class DocumentsWriterThreadPool { } catch (InterruptedException ie) { throw new ThreadInterruptedException(ie); } - - pauseAllThreads(); + + assert !globalLock; globalLock = true; + + pauseAllThreads(); + } finally { lock.unlock(); } + final ThreadState[] localAllThreads = allThreadStates; // all threads are idle now - + boolean success = false; try { - final ThreadState[] localAllThreads = allThreadStates; - result = task.process(new Iterator() { int i = 0; @@ -151,8 +192,18 @@ abstract class DocumentsWriterThreadPool { throw new UnsupportedOperationException("remove() not supported."); } }); + success = true; return result; } finally { + boolean abort = false; + if (!success) { + for (ThreadState state : localAllThreads) { + if (state.perThread.aborting) { + abort = true; + } + } + } + lock.lock(); try { try { @@ -168,6 +219,10 @@ abstract class DocumentsWriterThreadPool { lock.unlock(); } + if (!aborting && abort) { + documentsWriter.abort(); + } + } } @@ -182,13 +237,12 @@ abstract class DocumentsWriterThreadPool { } finally { boolean abort = false; if (!success && state.perThread.aborting) { - state.perThread.aborting = false; abort = true; } returnDocumentsWriterPerThread(state, task.doClearThreadBindings()); - if (abort) { + if (!aborting && abort) { documentsWriter.abort(); } } @@ -222,13 +276,15 @@ abstract class DocumentsWriterThreadPool { ThreadState threadState = selectThreadState(Thread.currentThread(), documentsWriter, doc); try { - while (!threadState.isIdle || globalLock || aborting) { + while (!threadState.isIdle || globalLock || aborting || threadState.perThread.aborting) { threadStateAvailable.await(); } } catch (InterruptedException ie) { throw new ThreadInterruptedException(ie); } + assert threadState.isIdle; + threadState.isIdle = false; threadState.start(); diff --git a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java index fa8bb9150f2..b5aa6a83501 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Set; import java.util.List; import java.util.Map; @@ -422,7 +423,7 @@ final class IndexFileDeleter { public void checkpoint(SegmentInfos segmentInfos, boolean isCommit) throws IOException { if (infoStream != null) { - message("now checkpoint \"" + segmentInfos.getCurrentSegmentFileName() + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]"); + message("now checkpoint \"" + segmentInfos + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]"); } // Try again now to delete any previously un-deletable @@ -442,18 +443,6 @@ final class IndexFileDeleter { // Decref files for commits that were deleted by the policy: deleteCommits(); } else { - - final List docWriterFiles; - if (docWriter != null) { - docWriterFiles = docWriter.openFiles(); - if (docWriterFiles != null) - // We must incRef these files before decRef'ing - // last files to make sure we don't accidentally - // delete them: - incRef(docWriterFiles); - } else - docWriterFiles = null; - // DecRef old files from the last checkpoint, if any: int size = lastFiles.size(); if (size > 0) { @@ -465,8 +454,6 @@ final class IndexFileDeleter { // Save files so we can decr on next checkpoint/commit: lastFiles.add(segmentInfos.files(directory, false)); - if (docWriterFiles != null) - lastFiles.add(docWriterFiles); } } diff --git a/lucene/src/java/org/apache/lucene/index/IndexReaderPool.java b/lucene/src/java/org/apache/lucene/index/IndexReaderPool.java new file mode 100644 index 00000000000..0d7752d79a4 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/IndexReaderPool.java @@ -0,0 +1,263 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.apache.lucene.store.BufferedIndexInput; +import org.apache.lucene.store.Directory; + +/** Holds shared SegmentReader instances. IndexWriter uses + * SegmentReaders for 1) applying deletes, 2) doing + * merges, 3) handing out a real-time reader. This pool + * reuses instances of the SegmentReaders in all these + * places if it is in "near real-time mode" (getReader() + * has been called on this instance). */ +public class IndexReaderPool { + + private final Map readerMap = new HashMap(); + + private final Directory directory; + private final IndexWriterConfig config; + private final IndexWriter writer; + + public IndexReaderPool(IndexWriter writer, Directory directory, IndexWriterConfig config) { + this.directory = directory; + this.config = config; + this.writer = writer; + } + + /** Forcefully clear changes for the specified segments, + * and remove from the pool. This is called on successful merge. */ + synchronized void clear(SegmentInfos infos) throws IOException { + if (infos == null) { + for (Map.Entry ent: readerMap.entrySet()) { + ent.getValue().hasChanges = false; + } + } else { + for (final SegmentInfo info: infos) { + if (readerMap.containsKey(info)) { + readerMap.get(info).hasChanges = false; + } + } + } + } + + /** + * Release the segment reader (i.e. decRef it and close if there + * are no more references. + * @param sr + * @throws IOException + */ + public synchronized void release(SegmentReader sr) throws IOException { + release(sr, false); + } + + /** + * Release the segment reader (i.e. decRef it and close if there + * are no more references. + * @param sr + * @throws IOException + */ + public synchronized void release(SegmentReader sr, boolean drop) throws IOException { + + final boolean pooled = readerMap.containsKey(sr.getSegmentInfo()); + + assert !pooled | readerMap.get(sr.getSegmentInfo()) == sr; + + // Drop caller's ref; for an external reader (not + // pooled), this decRef will close it + sr.decRef(); + + if (pooled && (drop || (!writer.poolReaders && sr.getRefCount() == 1))) { + + // We are the last ref to this reader; since we're + // not pooling readers, we release it: + readerMap.remove(sr.getSegmentInfo()); + + // nocommit + //assert !sr.hasChanges || Thread.holdsLock(IndexWriter.this); + + // Drop our ref -- this will commit any pending + // changes to the dir + boolean success = false; + try { + sr.close(); + success = true; + } finally { + if (!success && sr.hasChanges) { + // Abandon the changes & retry closing: + sr.hasChanges = false; + try { + sr.close(); + } catch (Throwable ignore) { + // Keep throwing original exception + } + } + } + } + } + + /** Remove all our references to readers, and commits + * any pending changes. */ + synchronized void close() throws IOException { + Iterator> iter = readerMap.entrySet().iterator(); + while (iter.hasNext()) { + + Map.Entry ent = iter.next(); + + SegmentReader sr = ent.getValue(); + if (sr.hasChanges) { + assert writer.infoIsLive(sr.getSegmentInfo()); + sr.startCommit(); + boolean success = false; + try { + sr.doCommit(null); + success = true; + } finally { + if (!success) { + sr.rollbackCommit(); + } + } + } + + iter.remove(); + + // NOTE: it is allowed that this decRef does not + // actually close the SR; this can happen when a + // near real-time reader is kept open after the + // IndexWriter instance is closed + sr.decRef(); + } + } + + /** + * Commit all segment reader in the pool. + * @throws IOException + */ + synchronized void commit() throws IOException { + for (Map.Entry ent : readerMap.entrySet()) { + + SegmentReader sr = ent.getValue(); + if (sr.hasChanges) { + assert writer.infoIsLive(sr.getSegmentInfo()); + sr.startCommit(); + boolean success = false; + try { + sr.doCommit(null); + success = true; + } finally { + if (!success) { + sr.rollbackCommit(); + } + } + } + } + } + + /** + * Returns a ref to a clone. NOTE: this clone is not + * enrolled in the pool, so you should simply close() + * it when you're done (ie, do not call release()). + */ + public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor) throws IOException { + SegmentReader sr = get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, termInfosIndexDivisor); + try { + return (SegmentReader) sr.clone(true); + } finally { + sr.decRef(); + } + } + + /** + * Obtain a SegmentReader from the readerPool. The reader + * must be returned by calling {@link #release(SegmentReader)} + * @see #release(SegmentReader) + * @param info + * @param doOpenStores + * @throws IOException + */ + public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores) throws IOException { + return get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, config.getReaderTermsIndexDivisor()); + } + + /** + * Obtain a SegmentReader from the readerPool. The reader + * must be returned by calling {@link #release(SegmentReader)} + * + * @see #release(SegmentReader) + * @param info + * @param doOpenStores + * @param readBufferSize + * @param termsIndexDivisor + * @throws IOException + */ + public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, int readBufferSize, int termsIndexDivisor) throws IOException { + + if (writer.poolReaders) { + readBufferSize = BufferedIndexInput.BUFFER_SIZE; + } + + SegmentReader sr = readerMap.get(info); + if (sr == null) { + // TODO: we may want to avoid doing this while + // synchronized + // Returns a ref, which we xfer to readerMap: + sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor, config.getCodecProvider()); + + if (info.dir == directory) { + // Only pool if reader is not external + readerMap.put(info, sr); + } + } else { + if (doOpenStores) { + sr.openDocStores(); + } + if (termsIndexDivisor != -1) { + // If this reader was originally opened because we + // needed to merge it, we didn't load the terms + // index. But now, if the caller wants the terms + // index (eg because it's doing deletes, or an NRT + // reader is being opened) we ask the reader to + // load its terms index. + sr.loadTermsIndex(termsIndexDivisor); + } + } + + // Return a ref to our caller + if (info.dir == directory) { + // Only incRef if we pooled (reader is not external) + sr.incRef(); + } + return sr; + } + + // Returns a ref + public synchronized SegmentReader getIfExists(SegmentInfo info) throws IOException { + SegmentReader sr = readerMap.get(info); + if (sr != null) { + sr.incRef(); + } + return sr; + } +} + + diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 60ddde46f0e..f527da2e711 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -271,8 +271,7 @@ public class IndexWriter implements Closeable { volatile SegmentInfos pendingCommit; // set when a commit is pending (after prepareCommit() & before commit()) volatile long pendingCommitChangeCount; - // nocommit - private - SegmentInfos segmentInfos = new SegmentInfos(); // the segments + private final SegmentInfos segmentInfos = new SegmentInfos(); // the segments private DocumentsWriter docWriter; //nocommit - private @@ -304,7 +303,7 @@ public class IndexWriter implements Closeable { private int flushCount; private int flushDeletesCount; - final ReaderPool readerPool = new ReaderPool(); + final IndexReaderPool readerPool; // This is a "write once" variable (like the organic dye // on a DVD-R that may or may not be heated by a laser and @@ -315,7 +314,7 @@ public class IndexWriter implements Closeable { // reuse SegmentReader instances internally for applying // deletes, doing merges, and reopening near real-time // readers. - private volatile boolean poolReaders; + volatile boolean poolReaders; // The instance that was passed to the constructor. It is saved only in order // to allow users to query an IndexWriter settings. @@ -427,246 +426,21 @@ public class IndexWriter implements Closeable { return r; } } - - /** Holds shared SegmentReader instances. IndexWriter uses - * SegmentReaders for 1) applying deletes, 2) doing - * merges, 3) handing out a real-time reader. This pool - * reuses instances of the SegmentReaders in all these - * places if it is in "near real-time mode" (getReader() - * has been called on this instance). */ - - class ReaderPool { - - private final Map readerMap = new HashMap(); - - /** Forcefully clear changes for the specified segments, - * and remove from the pool. This is called on successful merge. */ - synchronized void clear(SegmentInfos infos) throws IOException { - if (infos == null) { - for (Map.Entry ent: readerMap.entrySet()) { - ent.getValue().hasChanges = false; - } - } else { - for (final SegmentInfo info: infos) { - if (readerMap.containsKey(info)) { - readerMap.get(info).hasChanges = false; - } - } - } - } - - // used only by asserts - public synchronized boolean infoIsLive(SegmentInfo info) { - int idx = segmentInfos.indexOf(info); - assert idx != -1; - assert segmentInfos.get(idx) == info; - return true; - } - - public synchronized SegmentInfo mapToLive(SegmentInfo info) { - int idx = segmentInfos.indexOf(info); - if (idx != -1) { - info = segmentInfos.get(idx); - } - return info; - } - - /** - * Release the segment reader (i.e. decRef it and close if there - * are no more references. - * @param sr - * @throws IOException - */ - public synchronized void release(SegmentReader sr) throws IOException { - release(sr, false); - } - - /** - * Release the segment reader (i.e. decRef it and close if there - * are no more references. - * @param sr - * @throws IOException - */ - public synchronized void release(SegmentReader sr, boolean drop) throws IOException { - - final boolean pooled = readerMap.containsKey(sr.getSegmentInfo()); - - assert !pooled | readerMap.get(sr.getSegmentInfo()) == sr; - - // Drop caller's ref; for an external reader (not - // pooled), this decRef will close it - sr.decRef(); - - if (pooled && (drop || (!poolReaders && sr.getRefCount() == 1))) { - - // We are the last ref to this reader; since we're - // not pooling readers, we release it: - readerMap.remove(sr.getSegmentInfo()); - - assert !sr.hasChanges || Thread.holdsLock(IndexWriter.this); - - // Drop our ref -- this will commit any pending - // changes to the dir - boolean success = false; - try { - sr.close(); - success = true; - } finally { - if (!success && sr.hasChanges) { - // Abandon the changes & retry closing: - sr.hasChanges = false; - try { - sr.close(); - } catch (Throwable ignore) { - // Keep throwing original exception - } - } - } - } - } - - /** Remove all our references to readers, and commits - * any pending changes. */ - synchronized void close() throws IOException { - Iterator> iter = readerMap.entrySet().iterator(); - while (iter.hasNext()) { - - Map.Entry ent = iter.next(); - - SegmentReader sr = ent.getValue(); - if (sr.hasChanges) { - assert infoIsLive(sr.getSegmentInfo()); - sr.startCommit(); - boolean success = false; - try { - sr.doCommit(null); - success = true; - } finally { - if (!success) { - sr.rollbackCommit(); - } - } - } - - iter.remove(); - - // NOTE: it is allowed that this decRef does not - // actually close the SR; this can happen when a - // near real-time reader is kept open after the - // IndexWriter instance is closed - sr.decRef(); - } - } - - /** - * Commit all segment reader in the pool. - * @throws IOException - */ - synchronized void commit() throws IOException { - for (Map.Entry ent : readerMap.entrySet()) { - - SegmentReader sr = ent.getValue(); - if (sr.hasChanges) { - assert infoIsLive(sr.getSegmentInfo()); - sr.startCommit(); - boolean success = false; - try { - sr.doCommit(null); - success = true; - } finally { - if (!success) { - sr.rollbackCommit(); - } - } - } - } - } - - /** - * Returns a ref to a clone. NOTE: this clone is not - * enrolled in the pool, so you should simply close() - * it when you're done (ie, do not call release()). - */ - public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor) throws IOException { - SegmentReader sr = get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, termInfosIndexDivisor); - try { - return (SegmentReader) sr.clone(true); - } finally { - sr.decRef(); - } - } - - /** - * Obtain a SegmentReader from the readerPool. The reader - * must be returned by calling {@link #release(SegmentReader)} - * @see #release(SegmentReader) - * @param info - * @param doOpenStores - * @throws IOException - */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores) throws IOException { - return get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, config.getReaderTermsIndexDivisor()); - } - - /** - * Obtain a SegmentReader from the readerPool. The reader - * must be returned by calling {@link #release(SegmentReader)} - * - * @see #release(SegmentReader) - * @param info - * @param doOpenStores - * @param readBufferSize - * @param termsIndexDivisor - * @throws IOException - */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, int readBufferSize, int termsIndexDivisor) throws IOException { - - if (poolReaders) { - readBufferSize = BufferedIndexInput.BUFFER_SIZE; - } - - SegmentReader sr = readerMap.get(info); - if (sr == null) { - // TODO: we may want to avoid doing this while - // synchronized - // Returns a ref, which we xfer to readerMap: - sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor, codecs); - - if (info.dir == directory) { - // Only pool if reader is not external - readerMap.put(info, sr); - } - } else { - if (doOpenStores) { - sr.openDocStores(); - } - if (termsIndexDivisor != -1) { - // If this reader was originally opened because we - // needed to merge it, we didn't load the terms - // index. But now, if the caller wants the terms - // index (eg because it's doing deletes, or an NRT - // reader is being opened) we ask the reader to - // load its terms index. - sr.loadTermsIndex(termsIndexDivisor); - } - } - - // Return a ref to our caller - if (info.dir == directory) { - // Only incRef if we pooled (reader is not external) - sr.incRef(); - } - return sr; - } - - // Returns a ref - public synchronized SegmentReader getIfExists(SegmentInfo info) throws IOException { - SegmentReader sr = readerMap.get(info); - if (sr != null) { - sr.incRef(); - } - return sr; + + // used only by asserts + public synchronized boolean infoIsLive(SegmentInfo info) { + int idx = segmentInfos.indexOf(info); + assert idx != -1; + assert segmentInfos.get(idx) == info; + return true; + } + + public synchronized SegmentInfo mapToLive(SegmentInfo info) { + int idx = segmentInfos.indexOf(info); + if (idx != -1) { + info = segmentInfos.get(idx); } + return info; } /** @@ -934,6 +708,8 @@ public class IndexWriter implements Closeable { poolReaders = conf.getReaderPooling(); + this.readerPool = new IndexReaderPool(this, directory, config); + OpenMode mode = conf.getOpenMode(); boolean create; if (mode == OpenMode.CREATE) { @@ -1784,7 +1560,7 @@ public class IndexWriter implements Closeable { if (infoStream != null) { message("hit exception updating document"); } - + synchronized (this) { // If docWriter has some aborted files that were // never incref'd, then we clean them up here @@ -2419,6 +2195,15 @@ public class IndexWriter implements Closeable { deleter.checkpoint(segmentInfos, false); } + synchronized void addNewSegment(SegmentInfo newSegment) throws IOException { + segmentInfos.add(newSegment); + checkpoint(); + } + + boolean useCompoundFile(SegmentInfo segmentInfo) { + return mergePolicy.useCompoundFile(segmentInfos, segmentInfo); + } + private synchronized void resetMergeExceptions() { mergeExceptions = new ArrayList(); mergeGen++; @@ -2793,7 +2578,7 @@ public class IndexWriter implements Closeable { if (pendingCommit != null) { try { if (infoStream != null) - message("commit: pendingCommit != null"); + message("commit: pendingCommit != null"); pendingCommit.finishCommit(directory); if (infoStream != null) message("commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\""); @@ -2828,25 +2613,37 @@ public class IndexWriter implements Closeable { protected final void flush(boolean triggerMerge, boolean flushDeletes) throws CorruptIndexException, IOException { // We can be called during close, when closing==true, so we must pass false to ensureOpen: ensureOpen(false); - if (doFlush(flushDeletes) && triggerMerge) + + doBeforeFlush(); + + if (flushDeletes) { + if (applyDeletes()) { + checkpoint(); + } + } + boolean maybeMerge = false; + boolean success = false; + try { + maybeMerge = docWriter.flushAllThreads(flushDeletes) && triggerMerge; + success = true; + } finally { + if (!success) { + synchronized (this) { + // If docWriter has some aborted files that were + // never incref'd, then we clean them up here + final Collection files = docWriter.abortedFiles(); + if (files != null) { + deleter.deleteNewFiles(files); + } + } + } + } + + doAfterFlush(); + + if (maybeMerge) { maybeMerge(); - } - - // TODO: this method should not have to be entirely - // synchronized, ie, merges should be allowed to commit - // even while a flush is happening - private synchronized final boolean doFlush(boolean flushDeletes) throws CorruptIndexException, IOException { - return docWriter.flushAllThreads(flushDeletes); - // nocommit -// try { -// try { -// return doFlushInternal(flushDocStores, flushDeletes); -// } finally { -// docWriter.balanceRAM(); -// } -// } finally { -// docWriter.clearFlushPending(); -// } + } } /** Expert: Return the total size of all index files currently cached in memory. diff --git a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java index 2dfc1176e9e..5b921934554 100644 --- a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java @@ -79,9 +79,6 @@ final class StoredFieldsWriter { state.flushedFiles.add(fieldsName); state.flushedFiles.add(fieldsIdxName); - docWriter.removeOpenFile(fieldsName); - docWriter.removeOpenFile(fieldsIdxName); - if (4+((long) state.numDocs)*8 != state.directory.fileLength(fieldsIdxName)) { throw new RuntimeException("after flush: fdx size mismatch: " + state.numDocs + " docs vs " + state.directory.fileLength(fieldsIdxName) + " length in bytes of " + fieldsIdxName + " file exists?=" + state.directory.fileExists(fieldsIdxName)); } @@ -96,8 +93,6 @@ final class StoredFieldsWriter { fieldsWriter = new FieldsWriter(docWriter.directory, segment, fieldInfos); - docWriter.addOpenFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); - docWriter.addOpenFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); lastDocID = 0; } } diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java index 6fadbf36a44..03c2b4189dc 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java @@ -75,10 +75,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { state.flushedFiles.add(fldName); state.flushedFiles.add(docName); - docWriter.removeOpenFile(idxName); - docWriter.removeOpenFile(fldName); - docWriter.removeOpenFile(docName); - lastDocID = 0; } @@ -105,7 +101,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { } } - void initTermVectorsWriter() throws IOException { + private final void initTermVectorsWriter() throws IOException { if (tvx == null) { final String segment = docWriter.getSegment(); @@ -128,10 +124,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { tvd.writeInt(TermVectorsReader.FORMAT_CURRENT); tvf.writeInt(TermVectorsReader.FORMAT_CURRENT); - docWriter.addOpenFile(idxName); - docWriter.addOpenFile(fldName); - docWriter.addOpenFile(docName); - lastDocID = 0; } } @@ -146,7 +138,8 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { fill(docState.docID); // Append term vectors to the real outputs: - tvx.writeLong(tvd.getFilePointer()); + long pointer = tvd.getFilePointer(); + tvx.writeLong(pointer); tvx.writeLong(tvf.getFilePointer()); tvd.writeVInt(numVectorFields); if (numVectorFields > 0) { @@ -198,7 +191,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { } lastDocID = 0; - + reset(); } int numVectorFields; diff --git a/lucene/src/java/org/apache/lucene/index/TermsHash.java b/lucene/src/java/org/apache/lucene/index/TermsHash.java index 09ce7e64b1f..98849cd46f7 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsHash.java +++ b/lucene/src/java/org/apache/lucene/index/TermsHash.java @@ -82,9 +82,12 @@ final class TermsHash extends InvertedDocConsumer { @Override public void abort() { reset(); - consumer.abort(); - if (nextTermsHash != null) { - nextTermsHash.abort(); + try { + consumer.abort(); + } finally { + if (nextTermsHash != null) { + nextTermsHash.abort(); + } } } diff --git a/lucene/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/src/java/org/apache/lucene/store/FilterDirectory.java new file mode 100644 index 00000000000..30c860b2f57 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/store/FilterDirectory.java @@ -0,0 +1,111 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +public abstract class FilterDirectory extends Directory { + private final Directory delegate; + + public FilterDirectory(Directory delegate) { + this.delegate = delegate; + } + + @Override + public String[] listAll() throws IOException { + return delegate.listAll(); + } + + @Override + public boolean fileExists(String name) throws IOException { + return delegate.fileExists(name); + } + + @Override + public long fileModified(String name) throws IOException { + return delegate.fileModified(name); + } + + @Override + public void touchFile(String name) throws IOException { + delegate.touchFile(name); + } + + @Override + public void deleteFile(String name) throws IOException { + delegate.deleteFile(name); + } + + @Override + public long fileLength(String name) throws IOException { + return delegate.fileLength(name); + } + + @Override + public IndexOutput createOutput(String name) throws IOException { + return delegate.createOutput(name); + } + + @Override + public IndexInput openInput(String name) throws IOException { + return delegate.openInput(name); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Deprecated @Override + public void sync(String name) throws IOException { // TODO 4.0 kill me + delegate.sync(name); + } + + public void sync(Collection names) throws IOException { // TODO 4.0 make me abstract + delegate.sync(names); + } + + public IndexInput openInput(String name, int bufferSize) throws IOException { + return delegate.openInput(name, bufferSize); + } + + public Lock makeLock(String name) { + return delegate.makeLock(name); + } + + public void clearLock(String name) throws IOException { + delegate.clearLock(name); + } + + public void setLockFactory(LockFactory lockFactory) { + delegate.setLockFactory(lockFactory); + } + + public LockFactory getLockFactory() { + return delegate.getLockFactory(); + } + + public String getLockID() { + return delegate.getLockID(); + } + + public void copy(Directory to, String src, String dest) throws IOException { + delegate.copy(to, src, dest); + } +} diff --git a/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java b/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java index f354c284671..c3edcbaffea 100644 --- a/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java +++ b/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java @@ -48,7 +48,7 @@ public class TestConcurrentMergeScheduler extends LuceneTestCase { if (doFail && Thread.currentThread().getName().equals("main")) { StackTraceElement[] trace = new Exception().getStackTrace(); for (int i = 0; i < trace.length; i++) { - if ("doFlush".equals(trace[i].getMethodName())) { + if ("flush".equals(trace[i].getMethodName())) { hitExc = true; throw new IOException("now failing during flush"); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index 3ba715420d2..0f3e4e05bd9 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -2351,7 +2351,7 @@ public class TestIndexWriter extends LuceneTestCase { StackTraceElement[] trace = new Exception().getStackTrace(); for (int i = 0; i < trace.length; i++) { if ("abort".equals(trace[i].getMethodName()) || - "flushDocument".equals(trace[i].getMethodName())) { + "finishDocument".equals(trace[i].getMethodName())) { if (onlyOnce) doFail = false; //System.out.println(Thread.currentThread().getName() + ": now fail"); @@ -2419,7 +2419,7 @@ public class TestIndexWriter extends LuceneTestCase { for(int i=0;i 5); + assertTrue("flush should have occurred and files created", dir.listAll().length > 0); // After rollback, IW should remove all files writer.rollback(); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index a22f583af4c..5528dffd0ae 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -189,8 +189,9 @@ public class TestIndexWriterExceptions extends LuceneTestCase { threads[i].join(); for(int i=0;i Date: Tue, 27 Jul 2010 20:36:51 +0000 Subject: [PATCH 007/200] Setting svn:eol-style to native for new files in realtime branch git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@979858 13f79535-47bb-0310-9956-ffa450edef68 From 30bec40e58ded2de26e313225084bc7081558b78 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Wed, 28 Jul 2010 01:05:36 +0000 Subject: [PATCH 008/200] Make benchmarker contrib compile in realtime branch. Now all contribs compile and their tests pass. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@979921 13f79535-47bb-0310-9956-ffa450edef68 --- .../byTask/tasks/CreateIndexTask.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java b/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java index 2aa125efbc1..2ae6f63d1d1 100644 --- a/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java +++ b/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java @@ -22,6 +22,7 @@ import org.apache.lucene.benchmark.byTask.utils.Config; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LogMergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.MergePolicy; @@ -74,16 +75,16 @@ public class CreateIndexTask extends PerfTask { final String mergeScheduler = config.get("merge.scheduler", "org.apache.lucene.index.ConcurrentMergeScheduler"); if (mergeScheduler.equals(NoMergeScheduler.class.getName())) { - writer.setMergeScheduler(NoMergeScheduler.INSTANCE); + writer.getConfig().setMergeScheduler(NoMergeScheduler.INSTANCE); } else { try { - writer.setMergeScheduler(Class.forName(mergeScheduler).asSubclass(MergeScheduler.class).newInstance()); + writer.getConfig().setMergeScheduler(Class.forName(mergeScheduler).asSubclass(MergeScheduler.class).newInstance()); } catch (Exception e) { throw new RuntimeException("unable to instantiate class '" + mergeScheduler + "' as merge scheduler", e); } if (mergeScheduler.equals("org.apache.lucene.index.ConcurrentMergeScheduler")) { - ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) writer.getMergeScheduler(); + ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler(); int v = config.get("concurrent.merge.scheduler.max.thread.count", -1); if (v != -1) { cms.setMaxThreadCount(v); @@ -99,17 +100,19 @@ public class CreateIndexTask extends PerfTask { "org.apache.lucene.index.LogByteSizeMergePolicy"); boolean isCompound = config.get("compound", true); if (mergePolicy.equals(NoMergePolicy.class.getName())) { - writer.setMergePolicy(isCompound ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES); + writer.getConfig().setMergePolicy(isCompound ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES); } else { try { - writer.setMergePolicy(Class.forName(mergePolicy).asSubclass(MergePolicy.class).newInstance()); + writer.getConfig().setMergePolicy(Class.forName(mergePolicy).asSubclass(MergePolicy.class).newInstance()); } catch (Exception e) { throw new RuntimeException("unable to instantiate class '" + mergePolicy + "' as merge policy", e); } - writer.setUseCompoundFile(isCompound); - writer.setMergeFactor(config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR)); + if (writer.getConfig().getMergePolicy() instanceof LogMergePolicy) { + ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(isCompound); + ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR)); + } } - writer.setMaxFieldLength(config.get("max.field.length",OpenIndexTask.DEFAULT_MAX_FIELD_LENGTH)); + writer.getConfig().setMaxFieldLength(config.get("max.field.length",OpenIndexTask.DEFAULT_MAX_FIELD_LENGTH)); final double ramBuffer = config.get("ram.flush.mb",OpenIndexTask.DEFAULT_RAM_FLUSH_MB); final int maxBuffered = config.get("max.buffered",OpenIndexTask.DEFAULT_MAX_BUFFERED); From 683568df65717e2dc8d090834fed813d9c9b277b Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Wed, 28 Jul 2010 06:14:13 +0000 Subject: [PATCH 009/200] More realtime branch cleanup. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@979954 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/lucene/ant/IndexTask.java | 1 - .../codecs/appending/TestAppendingCodec.java | 1 - .../org/apache/lucene/wordnet/Syns2Index.java | 1 - .../apache/lucene/index/DocumentsWriter.java | 6 +-- .../org/apache/lucene/index/IndexWriter.java | 54 ++++++++++--------- .../apache/lucene/index/LogMergePolicy.java | 22 -------- .../org/apache/lucene/index/MergePolicy.java | 6 --- .../apache/lucene/index/NoMergePolicy.java | 3 -- .../test/org/apache/lucene/TestSearch.java | 1 - .../lucene/TestSearchForDuplicates.java | 1 - .../apache/lucene/index/TestAddIndexes.java | 7 --- .../index/TestBackwardsCompatibility.java | 2 - .../lucene/index/TestDeletionPolicy.java | 12 ----- .../apache/lucene/index/TestFieldsReader.java | 1 - .../lucene/index/TestIndexFileDeleter.java | 1 - .../apache/lucene/index/TestIndexReader.java | 1 - .../lucene/index/TestIndexReaderClone.java | 1 - .../index/TestIndexReaderCloneNorms.java | 2 - .../apache/lucene/index/TestIndexWriter.java | 3 -- .../lucene/index/TestIndexWriterDelete.java | 1 - .../org/apache/lucene/index/TestLazyBug.java | 1 - .../lucene/index/TestLazyProxSkipping.java | 1 - .../index/TestNRTReaderWithThreads.java | 1 - .../lucene/index/TestNoMergePolicy.java | 3 -- .../org/apache/lucene/index/TestNorms.java | 2 - .../org/apache/lucene/index/TestOmitTf.java | 1 - .../lucene/index/TestStressIndexing2.java | 2 - .../lucene/index/TestTermVectorsReader.java | 1 - .../lucene/store/TestFileSwitchDirectory.java | 1 - .../apache/lucene/util/LuceneTestCaseJ4.java | 1 - 30 files changed, 31 insertions(+), 110 deletions(-) diff --git a/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java b/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java index aa0356bb94e..ef76424073e 100644 --- a/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java +++ b/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java @@ -288,7 +288,6 @@ public class IndexTask extends Task { create ? OpenMode.CREATE : OpenMode.APPEND); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundIndex); - lmp.setUseCompoundDocStore(useCompoundIndex); lmp.setMergeFactor(mergeFactor); IndexWriter writer = new IndexWriter(dir, conf); int totalFiles = 0; diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java index cef9ece0b54..9e3f2e088c2 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java @@ -135,7 +135,6 @@ public class TestAppendingCodec extends LuceneTestCase { cfg.setCodecProvider(new AppendingCodecProvider()); ((LogMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy)cfg.getMergePolicy()).setUseCompoundDocStore(false); IndexWriter writer = new IndexWriter(dir, cfg); Document doc = new Document(); doc.add(new Field("f", text, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS)); diff --git a/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java b/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java index 82242dd7608..437b7e98973 100644 --- a/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java +++ b/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java @@ -251,7 +251,6 @@ public class Syns2Index IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( Version.LUCENE_CURRENT, ana).setOpenMode(OpenMode.CREATE)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why? - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(true); // why? Iterator i1 = word2Nums.keySet().iterator(); while (i1.hasNext()) // for each word { diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index c1e3e8e4d0b..bcc2d4ed20a 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -369,17 +369,17 @@ final class DocumentsWriter { message("hit exception " + "reating compound file for newly flushed segment " + newSegment.name); } - indexWriter.deleter.deleteFile(IndexFileNames.segmentFileName(newSegment.name, "", + indexWriter.getIndexFileDeleter().deleteFile(IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); for (String file : perThread.flushState.flushedFiles) { - indexWriter.deleter.deleteFile(file); + indexWriter.getIndexFileDeleter().deleteFile(file); } } } for (String file : perThread.flushState.flushedFiles) { - indexWriter.deleter.deleteFile(file); + indexWriter.getIndexFileDeleter().deleteFile(file); } newSegment.setUseCompoundFile(true); diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index f527da2e711..1e43e21c973 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -17,35 +17,35 @@ package org.apache.lucene.index; * limitations under the License. */ +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.util.Constants; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.ThreadInterruptedException; import org.apache.lucene.util.Version; -import java.io.IOException; -import java.io.Closeable; -import java.io.PrintStream; -import java.util.List; -import java.util.Collection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Set; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Iterator; -import java.util.Map; - /** An IndexWriter creates and maintains an index. @@ -274,8 +274,8 @@ public class IndexWriter implements Closeable { private final SegmentInfos segmentInfos = new SegmentInfos(); // the segments private DocumentsWriter docWriter; - //nocommit - private - IndexFileDeleter deleter; + + private IndexFileDeleter deleter; private Set segmentsToOptimize = new HashSet(); // used by optimize to note those needing optimization @@ -290,8 +290,8 @@ public class IndexWriter implements Closeable { // Holds all SegmentInfo instances currently involved in // merges private HashSet mergingSegments = new HashSet(); - // nocommit - private - MergePolicy mergePolicy; + + private MergePolicy mergePolicy; // TODO 4.0: this should be made final once the setter is removed private /*final*/MergeScheduler mergeScheduler; private LinkedList pendingMerges = new LinkedList(); @@ -1100,6 +1100,10 @@ public class IndexWriter implements Closeable { return infoStream != null; } + final IndexFileDeleter getIndexFileDeleter() { + return deleter; + } + /** * Commits all changes to an index and closes all * associated files. Note that this may be a costly @@ -2189,8 +2193,7 @@ public class IndexWriter implements Closeable { * the index files referenced exist (correctly) in the * index directory. */ - // nocommit - private - synchronized void checkpoint() throws IOException { + private synchronized void checkpoint() throws IOException { changeCount++; deleter.checkpoint(segmentInfos, false); } @@ -2200,7 +2203,7 @@ public class IndexWriter implements Closeable { checkpoint(); } - boolean useCompoundFile(SegmentInfo segmentInfo) { + synchronized boolean useCompoundFile(SegmentInfo segmentInfo) { return mergePolicy.useCompoundFile(segmentInfos, segmentInfo); } @@ -3017,12 +3020,11 @@ public class IndexWriter implements Closeable { mergingSegments.add(merge.info); } - // nocommit - private static void setDiagnostics(SegmentInfo info, String source) { setDiagnostics(info, source, null); } - private static void setDiagnostics(SegmentInfo info, String source, Map details) { + static void setDiagnostics(SegmentInfo info, String source, Map details) { Map diagnostics = new HashMap(); diagnostics.put("source", source); diagnostics.put("lucene.version", Constants.LUCENE_VERSION); diff --git a/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java b/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java index 48651bcf018..8992b25c588 100644 --- a/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java +++ b/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java @@ -64,7 +64,6 @@ public abstract class LogMergePolicy extends MergePolicy { protected boolean calibrateSizeByDeletes = false; private boolean useCompoundFile = true; - private boolean useCompoundDocStore = true; public LogMergePolicy() { super(); @@ -121,27 +120,6 @@ public abstract class LogMergePolicy extends MergePolicy { return useCompoundFile; } - // Javadoc inherited - @Override - public boolean useCompoundDocStore(SegmentInfos infos) { - return useCompoundDocStore; - } - - /** Sets whether compound file format should be used for - * newly flushed and newly merged doc store - * segment files (term vectors and stored fields). */ - public void setUseCompoundDocStore(boolean useCompoundDocStore) { - this.useCompoundDocStore = useCompoundDocStore; - } - - /** Returns true if newly flushed and newly merge doc - * store segment files (term vectors and stored fields) - * are written in compound file format. @see - * #setUseCompoundDocStore */ - public boolean getUseCompoundDocStore() { - return useCompoundDocStore; - } - /** Sets whether the segment size should be calibrated by * the number of deletes when choosing segments for merge. */ public void setCalibrateSizeByDeletes(boolean calibrateSizeByDeletes) { diff --git a/lucene/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/src/java/org/apache/lucene/index/MergePolicy.java index 8589a772697..2ab2abf9a7d 100644 --- a/lucene/src/java/org/apache/lucene/index/MergePolicy.java +++ b/lucene/src/java/org/apache/lucene/index/MergePolicy.java @@ -292,10 +292,4 @@ public abstract class MergePolicy implements java.io.Closeable { * segment should use the compound file format. */ public abstract boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment); - - /** - * Returns true if the doc store files should use the - * compound file format. - */ - public abstract boolean useCompoundDocStore(SegmentInfos segments); } diff --git a/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java b/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java index 5ef014ee1fb..abe48822155 100644 --- a/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java +++ b/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java @@ -66,9 +66,6 @@ public final class NoMergePolicy extends MergePolicy { public MergeSpecification findMergesToExpungeDeletes(SegmentInfos segmentInfos) throws CorruptIndexException, IOException { return null; } - @Override - public boolean useCompoundDocStore(SegmentInfos segments) { return useCompoundFile; } - @Override public boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) { return useCompoundFile; } diff --git a/lucene/src/test/org/apache/lucene/TestSearch.java b/lucene/src/test/org/apache/lucene/TestSearch.java index 75d7ebeb642..06cbe4d2e34 100644 --- a/lucene/src/test/org/apache/lucene/TestSearch.java +++ b/lucene/src/test/org/apache/lucene/TestSearch.java @@ -76,7 +76,6 @@ public class TestSearch extends LuceneTestCase { IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(directory, conf); String[] docs = { diff --git a/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java b/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java index 9f11ef0edd5..22c6cead1fe 100644 --- a/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java +++ b/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java @@ -81,7 +81,6 @@ public class TestSearchForDuplicates extends LuceneTestCase { IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFiles); - lmp.setUseCompoundDocStore(useCompoundFiles); IndexWriter writer = new IndexWriter(directory, conf); final int MAX_DOCS = 225; diff --git a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java index 7f53710530e..5c07abf7b91 100755 --- a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -53,7 +53,6 @@ public class TestAddIndexes extends LuceneTestCase { writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file // add 40 documents in separate files addDocs(writer, 40); assertEquals(40, writer.maxDoc()); @@ -261,13 +260,11 @@ public class TestAddIndexes extends LuceneTestCase { writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file // add 140 documents in separate files addDocs(writer, 40); writer.close(); writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file addDocs(writer, 100); writer.close(); @@ -482,7 +479,6 @@ public class TestAddIndexes extends LuceneTestCase { writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); // add 30 documents in 3 segments for (int i = 0; i < 3; i++) { @@ -490,7 +486,6 @@ public class TestAddIndexes extends LuceneTestCase { writer.close(); writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(100)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); } assertEquals(30, writer.maxDoc()); @@ -504,7 +499,6 @@ public class TestAddIndexes extends LuceneTestCase { Directory dir = new MockRAMDirectory(); LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(100); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()) @@ -533,7 +527,6 @@ public class TestAddIndexes extends LuceneTestCase { lmp = new LogByteSizeMergePolicy(); lmp.setMinMergeMB(0.0001); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(4); writer = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) diff --git a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index f946f0c37fc..db17b938040 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -478,7 +478,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase { Directory dir = FSDirectory.open(new File(dirName)); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10); ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS); - ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<35;i++) { @@ -490,7 +489,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase { // open fresh writer so we get no prx file in the added segment conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10); ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS); - ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS); writer = new IndexWriter(dir, conf); addNoProxDoc(writer); writer.close(); diff --git a/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java b/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java index 4d22920dedb..41d9aa03a79 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java +++ b/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java @@ -207,7 +207,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setIndexDeletionPolicy(policy); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); writer.close(); @@ -221,7 +220,6 @@ public class TestDeletionPolicy extends LuceneTestCase { OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); for(int j=0;j<17;j++) { addDoc(writer); @@ -288,7 +286,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setMergeScheduler(new SerialMergeScheduler()); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<107;i++) { addDoc(writer); @@ -300,7 +297,6 @@ public class TestDeletionPolicy extends LuceneTestCase { OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); writer.optimize(); writer.close(); @@ -475,7 +471,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setMaxBufferedDocs(10); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<107;i++) { addDoc(writer); @@ -486,7 +481,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); writer.optimize(); writer.close(); @@ -527,7 +521,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setMaxBufferedDocs(10); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); for(int i=0;i<17;i++) { addDoc(writer); @@ -586,7 +579,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); writer.close(); Term searchTerm = new Term("content", "aaa"); @@ -598,7 +590,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); for(int j=0;j<17;j++) { addDoc(writer); @@ -619,7 +610,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); writer.optimize(); // this is a commit @@ -696,7 +686,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setMaxBufferedDocs(10); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); IndexWriter writer = new IndexWriter(dir, conf); writer.close(); Term searchTerm = new Term("content", "aaa"); @@ -710,7 +699,6 @@ public class TestDeletionPolicy extends LuceneTestCase { .setMaxBufferedDocs(10); lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(useCompoundFile); - lmp.setUseCompoundDocStore(useCompoundFile); writer = new IndexWriter(dir, conf); for(int j=0;j<17;j++) { addDoc(writer); diff --git a/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java index f49fd271dd4..6338ca2eb6c 100644 --- a/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java @@ -62,7 +62,6 @@ public class TestFieldsReader extends LuceneTestCase { fieldInfos.add(testDoc); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()); ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(false); IndexWriter writer = new IndexWriter(dir, conf); writer.addDocument(testDoc); writer.close(); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java b/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java index 06438fe315c..e95aac9c416 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java @@ -49,7 +49,6 @@ public class TestIndexFileDeleter extends LuceneTestCase { addDoc(writer, i); } ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); for(;i<45;i++) { addDoc(writer, i); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReader.java b/lucene/src/test/org/apache/lucene/index/TestIndexReader.java index c8410e23b1d..f40fbb95cd9 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReader.java @@ -555,7 +555,6 @@ public class TestIndexReader extends LuceneTestCase // add 1 documents with term : aaa writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); addDoc(writer, searchTerm.text()); writer.close(); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java b/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java index 1fac3feb1fc..0ff9c73b27a 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java @@ -488,7 +488,6 @@ public class TestIndexReaderClone extends LuceneTestCase { IndexWriter w = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer())); ((LogMergePolicy) w.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) w.getConfig().getMergePolicy()).setUseCompoundDocStore(false); Document doc = new Document(); doc.add(new Field("field", "yes it's stored", Field.Store.YES, Field.Index.ANALYZED)); w.addDocument(doc); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java b/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java index 3fd2dad651a..a1e16412b93 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java @@ -239,7 +239,6 @@ public class TestIndexReaderCloneNorms extends LuceneTestCase { LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(true); - lmp.setUseCompoundDocStore(true); iw.close(); } @@ -292,7 +291,6 @@ public class TestIndexReaderCloneNorms extends LuceneTestCase { LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(compound); - lmp.setUseCompoundDocStore(compound); IndexWriter iw = new IndexWriter(dir, conf); for (int i = 0; i < ndocs; i++) { iw.addDocument(newDoc()); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index 4c81c047f3a..3de1996c5ed 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -2014,7 +2014,6 @@ public class TestIndexWriter extends LuceneTestCase { LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setMergeFactor(2); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); Document doc = new Document(); String contents = "aa bb cc dd ee ff gg hh ii jj kk"; @@ -2050,7 +2049,6 @@ public class TestIndexWriter extends LuceneTestCase { writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); LogMergePolicy lmp2 = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp2.setUseCompoundFile(false); - lmp2.setUseCompoundDocStore(false); writer.optimize(); writer.close(); } @@ -4553,7 +4551,6 @@ public class TestIndexWriter extends LuceneTestCase { TEST_VERSION_CURRENT, new MockAnalyzer())); LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); writer.setInfoStream(new PrintStream(bos)); writer.addDocument(new Document()); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index 8604a980043..7710392df2a 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -644,7 +644,6 @@ public class TestIndexWriterDelete extends LuceneTestCase { TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)).setMaxBufferedDeleteTerms(2)); LogMergePolicy lmp = (LogMergePolicy) modifier.getConfig().getMergePolicy(); lmp.setUseCompoundFile(true); - lmp.setUseCompoundDocStore(true); dir.failOn(failure.reset()); diff --git a/lucene/src/test/org/apache/lucene/index/TestLazyBug.java b/lucene/src/test/org/apache/lucene/index/TestLazyBug.java index b7dc476ab20..fc8ccb32871 100755 --- a/lucene/src/test/org/apache/lucene/index/TestLazyBug.java +++ b/lucene/src/test/org/apache/lucene/index/TestLazyBug.java @@ -75,7 +75,6 @@ public class TestLazyBug extends LuceneTestCase { TEST_VERSION_CURRENT, new MockAnalyzer())); LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); for (int d = 1; d <= NUM_DOCS; d++) { Document doc = new Document(); diff --git a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java index 4c15e0834ad..72a3c2d1353 100755 --- a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java +++ b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java @@ -63,7 +63,6 @@ public class TestLazyProxSkipping extends LuceneTestCase { Directory directory = new SeekCountingDirectory(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); String content; diff --git a/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java b/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java index 7913becefd5..8f2809ceb09 100644 --- a/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java +++ b/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java @@ -35,7 +35,6 @@ public class TestNRTReaderWithThreads extends LuceneTestCase { IndexWriter writer = new IndexWriter(mainDir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); IndexReader reader = writer.getReader(); // start pooling readers reader.close(); RunThread[] indexThreads = new RunThread[4]; diff --git a/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java b/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java index f36c2e55df8..532634a1e86 100644 --- a/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java +++ b/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java @@ -35,16 +35,13 @@ public class TestNoMergePolicy extends LuceneTestCaseJ4 { assertNull(mp.findMerges(null)); assertNull(mp.findMergesForOptimize(null, 0, null)); assertNull(mp.findMergesToExpungeDeletes(null)); - assertFalse(mp.useCompoundDocStore(null)); assertFalse(mp.useCompoundFile(null, null)); mp.close(); } @Test public void testCompoundFiles() throws Exception { - assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundDocStore(null)); assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundFile(null, null)); - assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundDocStore(null)); assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundFile(null, null)); } diff --git a/lucene/src/test/org/apache/lucene/index/TestNorms.java b/lucene/src/test/org/apache/lucene/index/TestNorms.java index 94f047aef54..e5563dba659 100755 --- a/lucene/src/test/org/apache/lucene/index/TestNorms.java +++ b/lucene/src/test/org/apache/lucene/index/TestNorms.java @@ -151,7 +151,6 @@ public class TestNorms extends LuceneTestCase { LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(true); - lmp.setUseCompoundDocStore(true); iw.close(); } @@ -196,7 +195,6 @@ public class TestNorms extends LuceneTestCase { LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy(); lmp.setMergeFactor(3); lmp.setUseCompoundFile(compound); - lmp.setUseCompoundDocStore(compound); for (int i = 0; i < ndocs; i++) { iw.addDocument(newDoc()); } diff --git a/lucene/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/src/test/org/apache/lucene/index/TestOmitTf.java index cc6d321beaa..c40a24e3c3b 100644 --- a/lucene/src/test/org/apache/lucene/index/TestOmitTf.java +++ b/lucene/src/test/org/apache/lucene/index/TestOmitTf.java @@ -214,7 +214,6 @@ public class TestOmitTf extends LuceneTestCase { LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy(); lmp.setMergeFactor(2); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); Document d = new Document(); Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED); diff --git a/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java index 5a091c57e37..96f1a9e1ea6 100644 --- a/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java +++ b/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java @@ -147,7 +147,6 @@ public class TestStressIndexing2 extends MultiCodecTestCase { w.commit(); LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(mergeFactor); /*** w.setMaxMergeDocs(Integer.MAX_VALUE); @@ -200,7 +199,6 @@ public class TestStressIndexing2 extends MultiCodecTestCase { .setReaderPooling(doReaderPooling)); LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy(); lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(mergeFactor); threads = new IndexingThread[nThreads]; diff --git a/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java b/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java index ed411a1a883..170f766fc82 100644 --- a/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java @@ -95,7 +95,6 @@ public class TestTermVectorsReader extends LuceneTestCase { IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MyAnalyzer())); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); Document doc = new Document(); for(int i=0;i Date: Tue, 21 Dec 2010 23:49:03 +0000 Subject: [PATCH 010/200] Remove files that should have been deleted as part of the merge with trunk git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1051701 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene/index/BufferedDeletesInRAM.java | 70 --- .../apache/lucene/index/ByteBlockPool.java | 169 ------ .../index/DocumentsWriterRAMAllocator.java | 147 ----- .../apache/lucene/index/IndexReaderPool.java | 263 --------- .../apache/lucene/store/FilterDirectory.java | 111 ---- .../util/ThreadSafeCloneableSortedMap.java | 156 ------ .../apache/lucene/util/LuceneTestCaseJ4.java | 513 ------------------ 7 files changed, 1429 deletions(-) delete mode 100644 lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java delete mode 100644 lucene/src/java/org/apache/lucene/index/ByteBlockPool.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java delete mode 100644 lucene/src/java/org/apache/lucene/index/IndexReaderPool.java delete mode 100644 lucene/src/java/org/apache/lucene/store/FilterDirectory.java delete mode 100644 lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java delete mode 100644 lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java diff --git a/lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java b/lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java deleted file mode 100644 index 21ef5d52ed3..00000000000 --- a/lucene/src/java/org/apache/lucene/index/BufferedDeletesInRAM.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.apache.lucene.index; - -import java.util.TreeMap; - -import org.apache.lucene.search.Query; -import org.apache.lucene.util.ThreadSafeCloneableSortedMap; - -public class BufferedDeletesInRAM { - static class Delete { - int flushCount; - - public Delete(int flushCount) { - this.flushCount = flushCount; - } - } - - final static class DeleteTerm extends Delete { - final Term term; - - public DeleteTerm(Term term, int flushCount) { - super(flushCount); - this.term = term; - } - } - - final static class DeleteTerms extends Delete { - final Term[] terms; - - public DeleteTerms(Term[] terms, int flushCount) { - super(flushCount); - this.terms = terms; - } - } - - final static class DeleteQuery extends Delete { - final Query query; - - public DeleteQuery(Query query, int flushCount) { - super(flushCount); - this.query = query; - } - } - - final ThreadSafeCloneableSortedMap deletes = ThreadSafeCloneableSortedMap - .getThreadSafeSortedMap(new TreeMap()); - - final void addDeleteTerm(Term term, long sequenceID, int numThreadStates) { - deletes.put(sequenceID, new DeleteTerm(term, numThreadStates)); - } - - final void addDeleteTerms(Term[] terms, long sequenceID, int numThreadStates) { - deletes.put(sequenceID, new DeleteTerms(terms, numThreadStates)); - } - - final void addDeleteQuery(Query query, long sequenceID, int numThreadStates) { - deletes.put(sequenceID, new DeleteQuery(query, numThreadStates)); - } - - boolean hasDeletes() { - return !deletes.isEmpty(); - } - - void clear() { - deletes.clear(); - } - - int getNumDeletes() { - return this.deletes.size(); - } -} diff --git a/lucene/src/java/org/apache/lucene/index/ByteBlockPool.java b/lucene/src/java/org/apache/lucene/index/ByteBlockPool.java deleted file mode 100644 index 651c89d0ed5..00000000000 --- a/lucene/src/java/org/apache/lucene/index/ByteBlockPool.java +++ /dev/null @@ -1,169 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* Class that Posting and PostingVector use to write byte - * streams into shared fixed-size byte[] arrays. The idea - * is to allocate slices of increasing lengths For - * example, the first slice is 5 bytes, the next slice is - * 14, etc. We start by writing our bytes into the first - * 5 bytes. When we hit the end of the slice, we allocate - * the next slice and then write the address of the new - * slice into the last 4 bytes of the previous slice (the - * "forwarding address"). - * - * Each slice is filled with 0's initially, and we mark - * the end with a non-zero byte. This way the methods - * that are writing into the slice don't need to record - * its length and instead allocate a new slice once they - * hit a non-zero byte. */ - -import java.util.Arrays; -import org.apache.lucene.util.BytesRef; -import java.util.List; -import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF; -import org.apache.lucene.util.ArrayUtil; - -final class ByteBlockPool { - - abstract static class Allocator { - abstract void recycleByteBlocks(byte[][] blocks, int start, int end); - abstract void recycleByteBlocks(List blocks); - abstract byte[] getByteBlock(); - } - - public byte[][] buffers = new byte[10][]; - - int bufferUpto = -1; // Which buffer we are upto - public int byteUpto = DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; // Where we are in head buffer - - public byte[] buffer; // Current head buffer - public int byteOffset = -DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; // Current head offset - - private final Allocator allocator; - - public ByteBlockPool(Allocator allocator) { - this.allocator = allocator; - } - - public void reset() { - if (bufferUpto != -1) { - // We allocated at least one buffer - - for(int i=0;i 0) - // Recycle all but the first buffer - allocator.recycleByteBlocks(buffers, 1, 1+bufferUpto); - - // Re-use the first buffer - bufferUpto = 0; - byteUpto = 0; - byteOffset = 0; - buffer = buffers[0]; - } - } - - public void nextBuffer() { - if (1+bufferUpto == buffers.length) { - byte[][] newBuffers = new byte[ArrayUtil.oversize(buffers.length+1, - NUM_BYTES_OBJECT_REF)][]; - System.arraycopy(buffers, 0, newBuffers, 0, buffers.length); - buffers = newBuffers; - } - buffer = buffers[1+bufferUpto] = allocator.getByteBlock(); - bufferUpto++; - - byteUpto = 0; - byteOffset += DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE; - } - - public int newSlice(final int size) { - if (byteUpto > DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE-size) - nextBuffer(); - final int upto = byteUpto; - byteUpto += size; - buffer[byteUpto-1] = 16; - return upto; - } - - // Size of each slice. These arrays should be at most 16 - // elements (index is encoded with 4 bits). First array - // is just a compact way to encode X+1 with a max. Second - // array is the length of each slice, ie first slice is 5 - // bytes, next slice is 14 bytes, etc. - final static int[] nextLevelArray = {1, 2, 3, 4, 5, 6, 7, 8, 9, 9}; - final static int[] levelSizeArray = {5, 14, 20, 30, 40, 40, 80, 80, 120, 200}; - final static int FIRST_LEVEL_SIZE = levelSizeArray[0]; - - public int allocSlice(final byte[] slice, final int upto) { - - final int level = slice[upto] & 15; - final int newLevel = nextLevelArray[level]; - final int newSize = levelSizeArray[newLevel]; - - // Maybe allocate another block - if (byteUpto > DocumentsWriterRAMAllocator.BYTE_BLOCK_SIZE-newSize) - nextBuffer(); - - final int newUpto = byteUpto; - final int offset = newUpto + byteOffset; - byteUpto += newSize; - - // Copy forward the past 3 bytes (which we are about - // to overwrite with the forwarding address): - buffer[newUpto] = slice[upto-3]; - buffer[newUpto+1] = slice[upto-2]; - buffer[newUpto+2] = slice[upto-1]; - - // Write forwarding address at end of last slice: - slice[upto-3] = (byte) (offset >>> 24); - slice[upto-2] = (byte) (offset >>> 16); - slice[upto-1] = (byte) (offset >>> 8); - slice[upto] = (byte) offset; - - // Write new level: - buffer[byteUpto-1] = (byte) (16|newLevel); - - return newUpto+3; - } - - // Fill in a BytesRef from term's length & bytes encoded in - // byte block - final BytesRef setBytesRef(BytesRef term, int textStart) { - final byte[] bytes = term.bytes = buffers[textStart >> DocumentsWriterRAMAllocator.BYTE_BLOCK_SHIFT]; - int pos = textStart & DocumentsWriterRAMAllocator.BYTE_BLOCK_MASK; - if ((bytes[pos] & 0x80) == 0) { - // length is 1 byte - term.length = bytes[pos]; - term.offset = pos+1; - } else { - // length is 2 bytes - term.length = (bytes[pos]&0x7f) + ((bytes[pos+1]&0xff)<<7); - term.offset = pos+2; - } - assert term.length >= 0; - return term; - } -} - diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java deleted file mode 100644 index 0ece8dee33a..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterRAMAllocator.java +++ /dev/null @@ -1,147 +0,0 @@ -package org.apache.lucene.index; - -import java.text.NumberFormat; -import java.util.ArrayList; -import java.util.List; - -import org.apache.lucene.util.Constants; - -class DocumentsWriterRAMAllocator { - final ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE); - final ByteBlockAllocator perDocAllocator = new ByteBlockAllocator(PER_DOC_BLOCK_SIZE); - - - class ByteBlockAllocator extends ByteBlockPool.Allocator { - final int blockSize; - - ByteBlockAllocator(int blockSize) { - this.blockSize = blockSize; - } - - ArrayList freeByteBlocks = new ArrayList(); - - /* Allocate another byte[] from the shared pool */ - @Override - byte[] getByteBlock() { - final int size = freeByteBlocks.size(); - final byte[] b; - if (0 == size) { - b = new byte[blockSize]; - // Always record a block allocated, even if - // trackAllocations is false. This is necessary - // because this block will be shared between - // things that don't track allocations (term - // vectors) and things that do (freq/prox - // postings). - numBytesUsed += blockSize; - } else - b = freeByteBlocks.remove(size-1); - return b; - } - - /* Return byte[]'s to the pool */ - @Override - void recycleByteBlocks(byte[][] blocks, int start, int end) { - for(int i=start;i blocks) { - final int size = blocks.size(); - for(int i=0;i freeIntBlocks = new ArrayList(); - - /* Allocate another int[] from the shared pool */ - int[] getIntBlock() { - final int size = freeIntBlocks.size(); - final int[] b; - if (0 == size) { - b = new int[INT_BLOCK_SIZE]; - // Always record a block allocated, even if - // trackAllocations is false. This is necessary - // because this block will be shared between - // things that don't track allocations (term - // vectors) and things that do (freq/prox - // postings). - numBytesUsed += INT_BLOCK_SIZE*INT_NUM_BYTE; - } else - b = freeIntBlocks.remove(size-1); - return b; - } - - void bytesUsed(long numBytes) { - numBytesUsed += numBytes; - } - - /* Return int[]s to the pool */ - void recycleIntBlocks(int[][] blocks, int start, int end) { - for(int i=start;i. Say list - allocates ~2X size (2*POINTER). Integer is OBJ_HEADER - + int */ - final static int BYTES_PER_DEL_DOCID = 2*POINTER_NUM_BYTE + OBJECT_HEADER_BYTES + INT_NUM_BYTE; - - /* Rough logic: HashMap has an array[Entry] w/ varying - load factor (say 2 * POINTER). Entry is object w/ - Query key, Integer val, int hash, Entry next - (OBJ_HEADER + 3*POINTER + INT). Query we often - undercount (say 24 bytes). Integer is OBJ_HEADER + INT. */ - final static int BYTES_PER_DEL_QUERY = 5*POINTER_NUM_BYTE + 2*OBJECT_HEADER_BYTES + 2*INT_NUM_BYTE + 24; - - /* Initial chunks size of the shared byte[] blocks used to - store postings data */ - final static int BYTE_BLOCK_SHIFT = 15; - final static int BYTE_BLOCK_SIZE = 1 << BYTE_BLOCK_SHIFT; - final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1; - final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK; - - final static int MAX_TERM_LENGTH_UTF8 = BYTE_BLOCK_SIZE-2; - - /* Initial chunks size of the shared int[] blocks used to - store postings data */ - final static int INT_BLOCK_SHIFT = 13; - final static int INT_BLOCK_SIZE = 1 << INT_BLOCK_SHIFT; - final static int INT_BLOCK_MASK = INT_BLOCK_SIZE - 1; - - String toMB(long v) { - return nf.format(v/1024./1024.); - } -} diff --git a/lucene/src/java/org/apache/lucene/index/IndexReaderPool.java b/lucene/src/java/org/apache/lucene/index/IndexReaderPool.java deleted file mode 100644 index 0d7752d79a4..00000000000 --- a/lucene/src/java/org/apache/lucene/index/IndexReaderPool.java +++ /dev/null @@ -1,263 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.store.Directory; - -/** Holds shared SegmentReader instances. IndexWriter uses - * SegmentReaders for 1) applying deletes, 2) doing - * merges, 3) handing out a real-time reader. This pool - * reuses instances of the SegmentReaders in all these - * places if it is in "near real-time mode" (getReader() - * has been called on this instance). */ -public class IndexReaderPool { - - private final Map readerMap = new HashMap(); - - private final Directory directory; - private final IndexWriterConfig config; - private final IndexWriter writer; - - public IndexReaderPool(IndexWriter writer, Directory directory, IndexWriterConfig config) { - this.directory = directory; - this.config = config; - this.writer = writer; - } - - /** Forcefully clear changes for the specified segments, - * and remove from the pool. This is called on successful merge. */ - synchronized void clear(SegmentInfos infos) throws IOException { - if (infos == null) { - for (Map.Entry ent: readerMap.entrySet()) { - ent.getValue().hasChanges = false; - } - } else { - for (final SegmentInfo info: infos) { - if (readerMap.containsKey(info)) { - readerMap.get(info).hasChanges = false; - } - } - } - } - - /** - * Release the segment reader (i.e. decRef it and close if there - * are no more references. - * @param sr - * @throws IOException - */ - public synchronized void release(SegmentReader sr) throws IOException { - release(sr, false); - } - - /** - * Release the segment reader (i.e. decRef it and close if there - * are no more references. - * @param sr - * @throws IOException - */ - public synchronized void release(SegmentReader sr, boolean drop) throws IOException { - - final boolean pooled = readerMap.containsKey(sr.getSegmentInfo()); - - assert !pooled | readerMap.get(sr.getSegmentInfo()) == sr; - - // Drop caller's ref; for an external reader (not - // pooled), this decRef will close it - sr.decRef(); - - if (pooled && (drop || (!writer.poolReaders && sr.getRefCount() == 1))) { - - // We are the last ref to this reader; since we're - // not pooling readers, we release it: - readerMap.remove(sr.getSegmentInfo()); - - // nocommit - //assert !sr.hasChanges || Thread.holdsLock(IndexWriter.this); - - // Drop our ref -- this will commit any pending - // changes to the dir - boolean success = false; - try { - sr.close(); - success = true; - } finally { - if (!success && sr.hasChanges) { - // Abandon the changes & retry closing: - sr.hasChanges = false; - try { - sr.close(); - } catch (Throwable ignore) { - // Keep throwing original exception - } - } - } - } - } - - /** Remove all our references to readers, and commits - * any pending changes. */ - synchronized void close() throws IOException { - Iterator> iter = readerMap.entrySet().iterator(); - while (iter.hasNext()) { - - Map.Entry ent = iter.next(); - - SegmentReader sr = ent.getValue(); - if (sr.hasChanges) { - assert writer.infoIsLive(sr.getSegmentInfo()); - sr.startCommit(); - boolean success = false; - try { - sr.doCommit(null); - success = true; - } finally { - if (!success) { - sr.rollbackCommit(); - } - } - } - - iter.remove(); - - // NOTE: it is allowed that this decRef does not - // actually close the SR; this can happen when a - // near real-time reader is kept open after the - // IndexWriter instance is closed - sr.decRef(); - } - } - - /** - * Commit all segment reader in the pool. - * @throws IOException - */ - synchronized void commit() throws IOException { - for (Map.Entry ent : readerMap.entrySet()) { - - SegmentReader sr = ent.getValue(); - if (sr.hasChanges) { - assert writer.infoIsLive(sr.getSegmentInfo()); - sr.startCommit(); - boolean success = false; - try { - sr.doCommit(null); - success = true; - } finally { - if (!success) { - sr.rollbackCommit(); - } - } - } - } - } - - /** - * Returns a ref to a clone. NOTE: this clone is not - * enrolled in the pool, so you should simply close() - * it when you're done (ie, do not call release()). - */ - public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor) throws IOException { - SegmentReader sr = get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, termInfosIndexDivisor); - try { - return (SegmentReader) sr.clone(true); - } finally { - sr.decRef(); - } - } - - /** - * Obtain a SegmentReader from the readerPool. The reader - * must be returned by calling {@link #release(SegmentReader)} - * @see #release(SegmentReader) - * @param info - * @param doOpenStores - * @throws IOException - */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores) throws IOException { - return get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, config.getReaderTermsIndexDivisor()); - } - - /** - * Obtain a SegmentReader from the readerPool. The reader - * must be returned by calling {@link #release(SegmentReader)} - * - * @see #release(SegmentReader) - * @param info - * @param doOpenStores - * @param readBufferSize - * @param termsIndexDivisor - * @throws IOException - */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, int readBufferSize, int termsIndexDivisor) throws IOException { - - if (writer.poolReaders) { - readBufferSize = BufferedIndexInput.BUFFER_SIZE; - } - - SegmentReader sr = readerMap.get(info); - if (sr == null) { - // TODO: we may want to avoid doing this while - // synchronized - // Returns a ref, which we xfer to readerMap: - sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor, config.getCodecProvider()); - - if (info.dir == directory) { - // Only pool if reader is not external - readerMap.put(info, sr); - } - } else { - if (doOpenStores) { - sr.openDocStores(); - } - if (termsIndexDivisor != -1) { - // If this reader was originally opened because we - // needed to merge it, we didn't load the terms - // index. But now, if the caller wants the terms - // index (eg because it's doing deletes, or an NRT - // reader is being opened) we ask the reader to - // load its terms index. - sr.loadTermsIndex(termsIndexDivisor); - } - } - - // Return a ref to our caller - if (info.dir == directory) { - // Only incRef if we pooled (reader is not external) - sr.incRef(); - } - return sr; - } - - // Returns a ref - public synchronized SegmentReader getIfExists(SegmentInfo info) throws IOException { - SegmentReader sr = readerMap.get(info); - if (sr != null) { - sr.incRef(); - } - return sr; - } -} - - diff --git a/lucene/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/src/java/org/apache/lucene/store/FilterDirectory.java deleted file mode 100644 index 30c860b2f57..00000000000 --- a/lucene/src/java/org/apache/lucene/store/FilterDirectory.java +++ /dev/null @@ -1,111 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collection; - -public abstract class FilterDirectory extends Directory { - private final Directory delegate; - - public FilterDirectory(Directory delegate) { - this.delegate = delegate; - } - - @Override - public String[] listAll() throws IOException { - return delegate.listAll(); - } - - @Override - public boolean fileExists(String name) throws IOException { - return delegate.fileExists(name); - } - - @Override - public long fileModified(String name) throws IOException { - return delegate.fileModified(name); - } - - @Override - public void touchFile(String name) throws IOException { - delegate.touchFile(name); - } - - @Override - public void deleteFile(String name) throws IOException { - delegate.deleteFile(name); - } - - @Override - public long fileLength(String name) throws IOException { - return delegate.fileLength(name); - } - - @Override - public IndexOutput createOutput(String name) throws IOException { - return delegate.createOutput(name); - } - - @Override - public IndexInput openInput(String name) throws IOException { - return delegate.openInput(name); - } - - @Override - public void close() throws IOException { - delegate.close(); - } - - @Deprecated @Override - public void sync(String name) throws IOException { // TODO 4.0 kill me - delegate.sync(name); - } - - public void sync(Collection names) throws IOException { // TODO 4.0 make me abstract - delegate.sync(names); - } - - public IndexInput openInput(String name, int bufferSize) throws IOException { - return delegate.openInput(name, bufferSize); - } - - public Lock makeLock(String name) { - return delegate.makeLock(name); - } - - public void clearLock(String name) throws IOException { - delegate.clearLock(name); - } - - public void setLockFactory(LockFactory lockFactory) { - delegate.setLockFactory(lockFactory); - } - - public LockFactory getLockFactory() { - return delegate.getLockFactory(); - } - - public String getLockID() { - return delegate.getLockID(); - } - - public void copy(Directory to, String src, String dest) throws IOException { - delegate.copy(to, src, dest); - } -} diff --git a/lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java b/lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java deleted file mode 100644 index 7ff5acf0ad7..00000000000 --- a/lucene/src/java/org/apache/lucene/util/ThreadSafeCloneableSortedMap.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.apache.lucene.util; - -import java.util.*; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -public class ThreadSafeCloneableSortedMap implements SortedMap, Cloneable { - - private volatile SortedMap copy; - private Lock cloneLock = new ReentrantLock(); - private final SortedMap delegate; - - private ThreadSafeCloneableSortedMap(SortedMap delegate) {this.delegate = delegate;} - - public static ThreadSafeCloneableSortedMap getThreadSafeSortedMap( - SortedMap delegate) { - return new ThreadSafeCloneableSortedMap(delegate); - } - - public SortedMap getReadCopy() { - SortedMap m = copy; - if (m != null) { - return m; - } - - // we have to clone - cloneLock.lock(); - try { - // check again - maybe a different thread was faster - m = copy; - if (m != null) { - return m; - } - - // still no copy there - create one now - SortedMap clone = clone(delegate); - copy = clone; - return clone; - } finally { - cloneLock.unlock(); - } - - } - - protected SortedMap clone(SortedMap map) { - if (map instanceof TreeMap) { - return (TreeMap) ((TreeMap) map).clone(); - } - - throw new IllegalArgumentException(map.getClass() + " not supported. Overwrite clone(SortedMap map) in a custom subclass to support this map."); - } - - private abstract static class Task { - abstract T run(); - } - - private final T withLock(Task task) { - copy = null; - cloneLock.lock(); - try { - return task.run(); - } finally { - cloneLock.unlock(); - } - } - - @Override public Comparator comparator() { - return delegate.comparator(); - } - - @Override public SortedMap subMap(K fromKey, K toKey) { - return delegate.subMap(fromKey, toKey); - } - - @Override public SortedMap headMap(K toKey) { - return delegate.headMap(toKey); - } - - @Override public SortedMap tailMap(K fromKey) { - return delegate.tailMap(fromKey); - } - - @Override public K firstKey() { - return delegate.firstKey(); - } - - @Override public K lastKey() { - return delegate.lastKey(); - } - - @Override public int size() { - return delegate.size(); - } - - @Override public boolean isEmpty() { - return delegate.isEmpty(); - } - - @Override public boolean containsKey(Object key) { - return delegate.containsKey(key); - } - - @Override public boolean containsValue(Object value) { - return delegate.containsValue(value); - } - - @Override public V get(Object key) { - return delegate.get(key); - } - - @Override public V put(final K key, final V value) { - return withLock(new Task() { - @Override V run() {return delegate.put(key, value);} - }); - } - - @Override public V remove(final Object key) { - return withLock(new Task() { - @Override V run() {return delegate.remove(key);} - }); - } - - @Override public void putAll(final Map m) { - withLock(new Task() { - @Override V run() { - delegate.putAll(m); - return null; - } - }); - } - - @Override public void clear() { - withLock(new Task() { - @Override V run() { - delegate.clear(); - return null; - } - }); - } - - // - // nocommit : don't use these methods to modify the map. - // TODO implement Set and Collection that acquire lock for modifications - // - @Override public Set keySet() { - return delegate.keySet(); - } - - @Override public Collection values() { - return delegate.values(); - } - - @Override public Set> entrySet() { - return delegate.entrySet(); - } -} diff --git a/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java b/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java deleted file mode 100644 index 40ba07147b9..00000000000 --- a/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java +++ /dev/null @@ -1,513 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.ConcurrentMergeScheduler; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LogDocMergePolicy; -import org.apache.lucene.index.LogMergePolicy; -import org.apache.lucene.index.SerialMergeScheduler; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.FieldCache; -import org.apache.lucene.search.FieldCache.CacheEntry; -import org.apache.lucene.util.FieldCacheSanityChecker.Insanity; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; - -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestWatchman; -import org.junit.runners.model.FrameworkMethod; - -import java.io.File; -import java.io.PrintStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Random; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.WeakHashMap; -import java.util.Collections; -import java.lang.reflect.Method; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * Base class for all Lucene unit tests, Junit4 variant. - * Replaces LuceneTestCase. - *

- *

- *

- * If you - * override either setUp() or - * tearDown() in your unit test, make sure you - * call super.setUp() and - * super.tearDown() - *

- * - * @After - replaces setup - * @Before - replaces teardown - * @Test - any public method with this annotation is a test case, regardless - * of its name - *

- *

- * See Junit4 documentation for a complete list of features. - *

- * Import from org.junit rather than junit.framework. - *

- * You should be able to use this class anywhere you used LuceneTestCase - * if you annotate your derived class correctly with the annotations above - * @see #assertSaneFieldCaches(String) - */ - - -// If we really need functionality in runBare override from LuceneTestCase, -// we can introduce RunBareWrapper and override runChild, and add the -// @RunWith annotation as below. runChild will be called for -// every test. But the functionality we used to -// get from that override is provided by InterceptTestCaseEvents -//@RunWith(RunBareWrapper.class) -public class LuceneTestCaseJ4 { - - /** - * true iff tests are run in verbose mode. Note: if it is false, tests are not - * expected to print any messages. - */ - public static final boolean VERBOSE = Boolean.getBoolean("tests.verbose"); - - /** Use this constant when creating Analyzers and any other version-dependent stuff. - *

NOTE: Change this when development starts for new Lucene version: - */ - public static final Version TEST_VERSION_CURRENT = Version.LUCENE_40; - - /** Create indexes in this directory, optimally use a subdir, named after the test */ - public static final File TEMP_DIR; - static { - String s = System.getProperty("tempDir", System.getProperty("java.io.tmpdir")); - if (s == null) - throw new RuntimeException("To run tests, you need to define system property 'tempDir' or 'java.io.tmpdir'."); - TEMP_DIR = new File(s); - } - - // by default we randomly pick a different codec for - // each test case (non-J4 tests) and each test class (J4 - // tests) - /** Gets the codec to run tests with. */ - static final String TEST_CODEC = System.getProperty("tests.codec", "random"); - - /** - * A random multiplier which you should use when writing random tests: - * multiply it by the number of iterations - */ - public static final int RANDOM_MULTIPLIER = Integer.parseInt(System.getProperty("random.multiplier", "1")); - - private int savedBoolMaxClauseCount; - - private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null; - - /** Used to track if setUp and tearDown are called correctly from subclasses */ - private boolean setup; - - private static class UncaughtExceptionEntry { - public final Thread thread; - public final Throwable exception; - - public UncaughtExceptionEntry(Thread thread, Throwable exception) { - this.thread = thread; - this.exception = exception; - } - } - private List uncaughtExceptions = Collections.synchronizedList(new ArrayList()); - - // checks if class correctly annotated - private static final Object PLACEHOLDER = new Object(); - private static final Map,Object> checkedClasses = - Collections.synchronizedMap(new WeakHashMap,Object>()); - - // saves default codec: we do this statically as many build indexes in @beforeClass - private static String savedDefaultCodec; - private static String codec; - private static Codec preFlexSav; - - // returns current PreFlex codec - public static Codec installPreFlexRW() { - final Codec preFlex = CodecProvider.getDefault().lookup("PreFlex"); - if (preFlex != null) { - CodecProvider.getDefault().unregister(preFlex); - } - CodecProvider.getDefault().register(new PreFlexRWCodec()); - return preFlex; - } - - // returns current PreFlex codec - public static void restorePreFlex(Codec preFlex) { - Codec preFlexRW = CodecProvider.getDefault().lookup("PreFlex"); - if (preFlexRW != null) { - CodecProvider.getDefault().unregister(preFlexRW); - } - CodecProvider.getDefault().register(preFlex); - } - - @BeforeClass - public static void beforeClassLuceneTestCaseJ4() { - savedDefaultCodec = CodecProvider.getDefaultCodec(); - codec = TEST_CODEC; - if (codec.equals("random")) - codec = CodecProvider.CORE_CODECS[seedRnd.nextInt(CodecProvider.CORE_CODECS.length)]; - - // If we're running w/ PreFlex codec we must swap in the - // test-only PreFlexRW codec (since core PreFlex can - // only read segments): - if (codec.equals("PreFlex")) { - preFlexSav = installPreFlexRW(); - } - - CodecProvider.setDefaultCodec(codec); - } - - @AfterClass - public static void afterClassLuceneTestCaseJ4() { - // Restore read-only PreFlex codec: - if (codec.equals("PreFlex")) { - restorePreFlex(preFlexSav); - } - CodecProvider.setDefaultCodec(savedDefaultCodec); - } - - // This is how we get control when errors occur. - // Think of this as start/end/success/failed - // events. - @Rule - public final TestWatchman intercept = new TestWatchman() { - - @Override - public void failed(Throwable e, FrameworkMethod method) { - reportAdditionalFailureInfo(); - super.failed(e, method); - } - - @Override - public void starting(FrameworkMethod method) { - // set current method name for logging - LuceneTestCaseJ4.this.name = method.getName(); - // check if the current test's class annotated all test* methods with @Test - final Class clazz = LuceneTestCaseJ4.this.getClass(); - if (!checkedClasses.containsKey(clazz)) { - checkedClasses.put(clazz, PLACEHOLDER); - for (Method m : clazz.getMethods()) { - if (m.getName().startsWith("test") && m.getAnnotation(Test.class) == null) { - fail("In class '" + clazz.getName() + "' the method '" + m.getName() + "' is not annotated with @Test."); - } - } - } - super.starting(method); - } - - }; - - @Before - public void setUp() throws Exception { - Assert.assertFalse("ensure your tearDown() calls super.tearDown()!!!", setup); - setup = true; - savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread t, Throwable e) { - uncaughtExceptions.add(new UncaughtExceptionEntry(t, e)); - if (savedUncaughtExceptionHandler != null) - savedUncaughtExceptionHandler.uncaughtException(t, e); - } - }); - - ConcurrentMergeScheduler.setTestMode(); - savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); - seed = null; - } - - - /** - * Forcible purges all cache entries from the FieldCache. - *

- * This method will be called by tearDown to clean up FieldCache.DEFAULT. - * If a (poorly written) test has some expectation that the FieldCache - * will persist across test methods (ie: a static IndexReader) this - * method can be overridden to do nothing. - *

- * - * @see FieldCache#purgeAllCaches() - */ - protected void purgeFieldCache(final FieldCache fc) { - fc.purgeAllCaches(); - } - - protected String getTestLabel() { - return getClass().getName() + "." + getName(); - } - - @After - public void tearDown() throws Exception { - Assert.assertTrue("ensure your setUp() calls super.setUp()!!!", setup); - setup = false; - BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount); - try { - - if (!uncaughtExceptions.isEmpty()) { - System.err.println("The following exceptions were thrown by threads:"); - for (UncaughtExceptionEntry entry : uncaughtExceptions) { - System.err.println("*** Thread: " + entry.thread.getName() + " ***"); - entry.exception.printStackTrace(System.err); - } - fail("Some threads threw uncaught exceptions!"); - } - - // calling assertSaneFieldCaches here isn't as useful as having test - // classes call it directly from the scope where the index readers - // are used, because they could be gc'ed just before this tearDown - // method is called. - // - // But it's better then nothing. - // - // If you are testing functionality that you know for a fact - // "violates" FieldCache sanity, then you should either explicitly - // call purgeFieldCache at the end of your test method, or refactor - // your Test class so that the inconsistant FieldCache usages are - // isolated in distinct test methods - assertSaneFieldCaches(getTestLabel()); - - if (ConcurrentMergeScheduler.anyUnhandledExceptions()) { - // Clear the failure so that we don't just keep - // failing subsequent test cases - ConcurrentMergeScheduler.clearUnhandledExceptions(); - fail("ConcurrentMergeScheduler hit unhandled exceptions"); - } - } finally { - purgeFieldCache(FieldCache.DEFAULT); - } - - Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler); - } - - /** - * Asserts that FieldCacheSanityChecker does not detect any - * problems with FieldCache.DEFAULT. - *

- * If any problems are found, they are logged to System.err - * (allong with the msg) when the Assertion is thrown. - *

- *

- * This method is called by tearDown after every test method, - * however IndexReaders scoped inside test methods may be garbage - * collected prior to this method being called, causing errors to - * be overlooked. Tests are encouraged to keep their IndexReaders - * scoped at the class level, or to explicitly call this method - * directly in the same scope as the IndexReader. - *

- * - * @see FieldCacheSanityChecker - */ - protected void assertSaneFieldCaches(final String msg) { - final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries(); - Insanity[] insanity = null; - try { - try { - insanity = FieldCacheSanityChecker.checkSanity(entries); - } catch (RuntimeException e) { - dumpArray(msg + ": FieldCache", entries, System.err); - throw e; - } - - assertEquals(msg + ": Insane FieldCache usage(s) found", - 0, insanity.length); - insanity = null; - } finally { - - // report this in the event of any exception/failure - // if no failure, then insanity will be null anyway - if (null != insanity) { - dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err); - } - - } - } - - /** - * Convinience method for logging an iterator. - * - * @param label String logged before/after the items in the iterator - * @param iter Each next() is toString()ed and logged on it's own line. If iter is null this is logged differnetly then an empty iterator. - * @param stream Stream to log messages to. - */ - public static void dumpIterator(String label, Iterator iter, - PrintStream stream) { - stream.println("*** BEGIN " + label + " ***"); - if (null == iter) { - stream.println(" ... NULL ..."); - } else { - while (iter.hasNext()) { - stream.println(iter.next().toString()); - } - } - stream.println("*** END " + label + " ***"); - } - - /** - * Convinience method for logging an array. Wraps the array in an iterator and delegates - * - * @see #dumpIterator(String,Iterator,PrintStream) - */ - public static void dumpArray(String label, Object[] objs, - PrintStream stream) { - Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator(); - dumpIterator(label, iter, stream); - } - - /** - * Returns a {@link Random} instance for generating random numbers during the test. - * The random seed is logged during test execution and printed to System.out on any failure - * for reproducing the test using {@link #newRandom(long)} with the recorded seed - * . - */ - public Random newRandom() { - if (seed != null) { - throw new IllegalStateException("please call LuceneTestCaseJ4.newRandom only once per test"); - } - this.seed = Long.valueOf(seedRnd.nextLong()); - if (VERBOSE) { - System.out.println("NOTE: random seed of testcase '" + getName() + "' is: " + this.seed); - } - return new Random(seed); - } - - /** - * Returns a {@link Random} instance for generating random numbers during the test. - * If an error occurs in the test that is not reproducible, you can use this method to - * initialize the number generator with the seed that was printed out during the failing test. - */ - public Random newRandom(long seed) { - if (this.seed != null) { - throw new IllegalStateException("please call LuceneTestCaseJ4.newRandom only once per test"); - } - System.out.println("WARNING: random seed of testcase '" + getName() + "' is fixed to: " + seed); - this.seed = Long.valueOf(seed); - return new Random(seed); - } - - private static final Map,Long> staticSeeds = - Collections.synchronizedMap(new WeakHashMap,Long>()); - - /** - * Returns a {@link Random} instance for generating random numbers from a beforeclass - * annotated method. - * The random seed is logged during test execution and printed to System.out on any failure - * for reproducing the test using {@link #newStaticRandom(Class, long)} with the recorded seed - * . - */ - public static Random newStaticRandom(Class clazz) { - Long seed = seedRnd.nextLong(); - staticSeeds.put(clazz, seed); - return new Random(seed); - } - - /** - * Returns a {@link Random} instance for generating random numbers from a beforeclass - * annotated method. - * If an error occurs in the test that is not reproducible, you can use this method to - * initialize the number generator with the seed that was printed out during the failing test. - */ - public static Random newStaticRandom(Class clazz, long seed) { - staticSeeds.put(clazz, Long.valueOf(seed)); - System.out.println("WARNING: random static seed of testclass '" + clazz + "' is fixed to: " + seed); - return new Random(seed); - } - - /** create a new index writer config with random defaults */ - public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) { - IndexWriterConfig c = new IndexWriterConfig(v, a); - if (r.nextBoolean()) { - c.setMergePolicy(new LogDocMergePolicy()); - } - if (r.nextBoolean()) { - c.setMergeScheduler(new SerialMergeScheduler()); - } - if (r.nextBoolean()) { - c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 1000)); - } - if (r.nextBoolean()) { - c.setTermIndexInterval(_TestUtil.nextInt(r, 1, 1000)); - } - - if (c.getMergePolicy() instanceof LogMergePolicy) { - LogMergePolicy logmp = (LogMergePolicy) c.getMergePolicy(); - logmp.setUseCompoundFile(r.nextBoolean()); - logmp.setCalibrateSizeByDeletes(r.nextBoolean()); - logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20)); - } - - c.setReaderPooling(r.nextBoolean()); - return c; - } - - public String getName() { - return this.name; - } - - /** Gets a resource from the classpath as {@link File}. This method should only be used, - * if a real file is needed. To get a stream, code should prefer - * {@link Class#getResourceAsStream} using {@code this.getClass()}. - */ - protected File getDataFile(String name) throws IOException { - try { - return new File(this.getClass().getResource(name).toURI()); - } catch (Exception e) { - throw new IOException("Cannot find resource: " + name); - } - } - - // We get here from InterceptTestCaseEvents on the 'failed' event.... - public void reportAdditionalFailureInfo() { - Long staticSeed = staticSeeds.get(getClass()); - if (staticSeed != null) { - System.out.println("NOTE: random static seed of testclass '" + getName() + "' was: " + staticSeed); - } - - if (TEST_CODEC.equals("random")) { - System.out.println("NOTE: random codec of testcase '" + getName() + "' was: " + codec); - } - - if (seed != null) { - System.out.println("NOTE: random seed of testcase '" + getName() + "' was: " + seed); - } - } - - // recorded seed - protected Long seed = null; - - // static members - private static final Random seedRnd = new Random(); - - private String name = ""; -} From 2bd8e508dead1e7c3fbb250928c8a1c1750fb4f0 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Thu, 13 Jan 2011 08:49:21 +0000 Subject: [PATCH 011/200] LUCENE-2324: More progress with concurrency and per-thread pool git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1058461 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 335 +++++++++--------- .../index/DocumentsWriterPerThread.java | 17 +- .../index/DocumentsWriterPerThreadPool.java | 81 +++++ .../index/DocumentsWriterThreadPool.java | 314 ---------------- .../org/apache/lucene/index/IndexWriter.java | 9 +- .../lucene/index/IndexWriterConfig.java | 72 ++-- ...readAffinityDocumentsWriterThreadPool.java | 76 ++-- 7 files changed, 341 insertions(+), 563 deletions(-) create mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java delete mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 2e45f6a21df..95f8842e91c 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.io.PrintStream; import java.util.Collection; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -28,6 +29,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; +import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; import org.apache.lucene.search.Query; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.AlreadyClosedException; @@ -107,7 +109,7 @@ final class DocumentsWriter { int numDocsInStore; // # docs written to doc stores boolean bufferIsFull; // True when it's time to write segment - private boolean closed; + private volatile boolean closed; PrintStream infoStream; int maxFieldLength = IndexWriterConfig.UNLIMITED_FIELD_LENGTH; @@ -115,12 +117,10 @@ final class DocumentsWriter { List newFiles; - private final DocumentsWriterThreadPool threadPool; final IndexWriter indexWriter; private AtomicInteger numDocsInRAM = new AtomicInteger(0); private AtomicLong ramUsed = new AtomicLong(0); - private int numDocumentsWriterPerThreads; static class DocState { DocumentsWriter docWriter; @@ -160,54 +160,71 @@ final class DocumentsWriter { private final FieldInfos fieldInfos; final BufferedDeletes bufferedDeletes; + final SegmentDeletes pendingDeletes; private final IndexWriter.FlushControl flushControl; - private final IndexingChain chain; + final IndexingChain chain; - DocumentsWriter(Directory directory, IndexWriter writer, IndexingChain chain, DocumentsWriterThreadPool indexerThreadPool, FieldInfos fieldInfos, BufferedDeletes bufferedDeletes) throws IOException { + final DocumentsWriterPerThreadPool perThreadPool; + + DocumentsWriter(Directory directory, IndexWriter writer, IndexingChain chain, DocumentsWriterPerThreadPool indexerThreadPool, FieldInfos fieldInfos, BufferedDeletes bufferedDeletes) throws IOException { this.directory = directory; this.indexWriter = writer; this.similarity = writer.getConfig().getSimilarity(); this.fieldInfos = fieldInfos; this.bufferedDeletes = bufferedDeletes; - this.threadPool = indexerThreadPool; + this.perThreadPool = indexerThreadPool; + this.pendingDeletes = new SegmentDeletes(); this.chain = chain; flushControl = writer.flushControl; + this.perThreadPool.initialize(this); } - boolean deleteQueries(Query... queries) { - final boolean doFlush = flushControl.waitUpdate(0, queries.length); - Iterator it = threadPool.getPerThreadIterator(); - while (it.hasNext()) { - it.next().deleteQueries(queries); + boolean deleteQueries(final Query... queries) throws IOException { + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + + boolean added = false; + while (threadsIterator.hasNext()) { + threadsIterator.next().perThread.deleteQueries(queries); + added = true; } - return doFlush; + + if (!added) { + synchronized(this) { + for (Query query : queries) { + pendingDeletes.addQuery(query, SegmentDeletes.MAX_INT); + } + } + } + + return true; } - boolean deleteQuery(Query query) { - final boolean doFlush = flushControl.waitUpdate(0, 1); - Iterator it = threadPool.getPerThreadIterator(); - while (it.hasNext()) { - it.next().deleteQuery(query); - } - return doFlush; + boolean deleteQuery(final Query query) throws IOException { + return deleteQueries(query); } - boolean deleteTerms(Term... terms) { - final boolean doFlush = flushControl.waitUpdate(0, terms.length); - Iterator it = threadPool.getPerThreadIterator(); - while (it.hasNext()) { - it.next().deleteTerms(terms); + boolean deleteTerms(final Term... terms) throws IOException { + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + + boolean added = false; + while (threadsIterator.hasNext()) { + threadsIterator.next().perThread.deleteTerms(terms); + added = true; } - return doFlush; + + if (!added) { + synchronized(this) { + for (Term term : terms) { + pendingDeletes.addTerm(term, SegmentDeletes.MAX_INT); + } + } + } + + return false; } - boolean deleteTerm(Term term, boolean skipWait) { - final boolean doFlush = flushControl.waitUpdate(0, 1, skipWait); - Iterator it = threadPool.getPerThreadIterator(); - while (it.hasNext()) { - it.next().deleteTerm(term); - } - return doFlush; + boolean deleteTerm(final Term term) throws IOException { + return deleteTerms(term); } public FieldInfos getFieldInfos() { @@ -224,25 +241,26 @@ final class DocumentsWriter { * here. */ synchronized void setInfoStream(PrintStream infoStream) { this.infoStream = infoStream; - Iterator it = threadPool.getPerThreadIterator(); - while (it.hasNext()) { - it.next().docState.infoStream = infoStream; - } + pushConfigChange(); } synchronized void setMaxFieldLength(int maxFieldLength) { this.maxFieldLength = maxFieldLength; - Iterator it = threadPool.getPerThreadIterator(); - while (it.hasNext()) { - it.next().docState.maxFieldLength = maxFieldLength; - } + pushConfigChange(); } synchronized void setSimilarity(Similarity similarity) { this.similarity = similarity; - Iterator it = threadPool.getPerThreadIterator(); + pushConfigChange(); + } + + private final void pushConfigChange() { + Iterator it = perThreadPool.getAllPerThreadsIterator(); while (it.hasNext()) { - it.next().docState.similarity = similarity; + DocumentsWriterPerThread perThread = it.next().perThread; + perThread.docState.infoStream = this.infoStream; + perThread.docState.maxFieldLength = this.maxFieldLength; + perThread.docState.similarity = this.similarity; } } @@ -300,14 +318,25 @@ final class DocumentsWriter { * currently buffered docs. This resets our state, * discarding any docs added since last flush. */ synchronized void abort() throws IOException { - if (infoStream != null) { - message("docWriter: abort"); - } - boolean success = false; - try { - threadPool.abort(); + try { + if (infoStream != null) { + message("docWriter: abort"); + } + + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + + while (threadsIterator.hasNext()) { + ThreadState perThread = threadsIterator.next(); + perThread.lock(); + try { + perThread.perThread.abort(); + } finally { + perThread.unlock(); + } + } + success = true; } finally { notifyAll(); @@ -324,16 +353,12 @@ final class DocumentsWriter { } // for testing - public SegmentDeletes getPendingDeletes() { - return null; - // nocommit - //return pendingDeletes; + public synchronized SegmentDeletes getPendingDeletes() { + return pendingDeletes; } - public boolean anyDeletions() { - // nocommit - return true; - //return pendingDeletes.any(); + public synchronized boolean anyDeletions() { + return pendingDeletes.any(); } synchronized void close() { @@ -341,52 +366,54 @@ final class DocumentsWriter { notifyAll(); } - synchronized DocumentsWriterPerThread newDocumentsWriterPerThread() { - DocumentsWriterPerThread perThread = new DocumentsWriterPerThread(directory, this, chain); - numDocumentsWriterPerThreads++; - return perThread; - } - boolean updateDocument(final Document doc, final Analyzer analyzer, final Term delTerm) throws CorruptIndexException, IOException { + ensureOpen(); - boolean flushed = threadPool.executePerThread(this, doc, - new DocumentsWriterThreadPool.PerThreadTask() { - @Override - public Boolean process(final DocumentsWriterPerThread perThread) throws IOException { - long perThreadRAMUsedBeforeAdd = perThread.bytesUsed(); - perThread.addDocument(doc, analyzer); + Collection flushedFiles = null; + SegmentInfo newSegment = null; - synchronized(DocumentsWriter.this) { - ensureOpen(); - if (delTerm != null) { - deleteTerm(delTerm, true); - } - perThread.commitDocument(); - numDocsInRAM.incrementAndGet(); - } + ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(), this, doc); + try { + DocumentsWriterPerThread dwpt = perThread.perThread; + long perThreadRAMUsedBeforeAdd = dwpt.bytesUsed(); + dwpt.addDocument(doc, analyzer); - if (finishAddDocument(perThread, perThreadRAMUsedBeforeAdd)) { - super.clearThreadBindings(); - return true; - } - return false; - } - }); + synchronized(DocumentsWriter.this) { + if (delTerm != null) { + deleteTerm(delTerm); + } + dwpt.commitDocument(); + numDocsInRAM.incrementAndGet(); + } - if (flushed) { - indexWriter.maybeMerge(); + newSegment = finishAddDocument(dwpt, perThreadRAMUsedBeforeAdd); + if (newSegment != null) { + perThreadPool.clearThreadBindings(perThread); + flushedFiles = new HashSet(); + flushedFiles.addAll(dwpt.flushState.flushedFiles); + } + + } finally { + perThread.unlock(); + } + + if (newSegment != null) { + finishFlushedSegment(newSegment, flushedFiles); return true; } return false; } - private final boolean finishAddDocument(DocumentsWriterPerThread perThread, + private final SegmentInfo finishAddDocument(DocumentsWriterPerThread perThread, long perThreadRAMUsedBeforeAdd) throws IOException { + SegmentInfo newSegment = null; + int numDocsPerThread = perThread.getNumDocsInRAM(); - boolean flushed = maybeFlushPerThread(perThread); - if (flushed) { + if (perThread.getNumDocsInRAM() == maxBufferedDocs) { + newSegment = perThread.flush(); + int oldValue = numDocsInRAM.get(); while (!numDocsInRAM.compareAndSet(oldValue, oldValue - numDocsPerThread)) { oldValue = numDocsInRAM.get(); @@ -399,81 +426,73 @@ final class DocumentsWriter { oldValue = ramUsed.get(); } - return flushed; - } - - private boolean flushSegment(DocumentsWriterPerThread perThread) throws IOException { - if (perThread.getNumDocsInRAM() == 0) { - return false; - } - - SegmentInfo newSegment = perThread.flush(); - newSegment.dir = indexWriter.getDirectory(); - - finishFlushedSegment(newSegment, perThread); - return true; - } - - private final boolean maybeFlushPerThread(DocumentsWriterPerThread perThread) throws IOException { - if (perThread.getNumDocsInRAM() == maxBufferedDocs) { - flushSegment(perThread); - assert perThread.getNumDocsInRAM() == 0; - - return true; - } - - return false; + return newSegment; } final boolean flushAllThreads(final boolean flushDeletes) throws IOException { - return threadPool.executeAllThreads(this, new DocumentsWriterThreadPool.AllThreadsTask() { - @Override - public Boolean process(Iterator threadsIterator) throws IOException { - boolean anythingFlushed = false; - - while (threadsIterator.hasNext()) { - DocumentsWriterPerThread perThread = threadsIterator.next(); - final int numDocs = perThread.getNumDocsInRAM(); - - // Always flush docs if there are any - boolean flushDocs = numDocs > 0; - - String segment = perThread.getSegment(); - - // If we are flushing docs, segment must not be null: - assert segment != null || !flushDocs; - - if (flushDocs) { - SegmentInfo newSegment = perThread.flush(); - newSegment.dir = indexWriter.getDirectory(); - - if (newSegment != null) { - anythingFlushed = true; - - IndexWriter.setDiagnostics(newSegment, "flush"); - finishFlushedSegment(newSegment, perThread); - } - } else if (flushDeletes) { - perThread.pushDeletes(null, indexWriter.segmentInfos); - } - } - - if (anythingFlushed) { - clearThreadBindings(); - numDocsInRAM.set(0); - } - - return anythingFlushed; + if (flushDeletes) { + if (indexWriter.segmentInfos.size() > 0 && pendingDeletes.any()) { + bufferedDeletes.pushDeletes(pendingDeletes, indexWriter.segmentInfos.lastElement(), true); + pendingDeletes.clear(); } - }); + } + + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + boolean anythingFlushed = false; + + while (threadsIterator.hasNext()) { + Collection flushedFiles = null; + SegmentInfo newSegment = null; + + ThreadState perThread = threadsIterator.next(); + perThread.lock(); + try { + DocumentsWriterPerThread dwpt = perThread.perThread; + final int numDocs = dwpt.getNumDocsInRAM(); + + // Always flush docs if there are any + boolean flushDocs = numDocs > 0; + + String segment = dwpt.getSegment(); + + // If we are flushing docs, segment must not be null: + assert segment != null || !flushDocs; + + if (flushDocs) { + newSegment = dwpt.flush(); + + if (newSegment != null) { + IndexWriter.setDiagnostics(newSegment, "flush"); + flushedFiles = new HashSet(); + flushedFiles.addAll(dwpt.flushState.flushedFiles); + dwpt.pushDeletes(newSegment, indexWriter.segmentInfos); + anythingFlushed = true; + perThreadPool.clearThreadBindings(perThread); + } + } else if (flushDeletes) { + dwpt.pushDeletes(null, indexWriter.segmentInfos); + } + } finally { + perThread.unlock(); + } + + if (newSegment != null) { + // important do unlock the perThread before finishFlushedSegment + // is called to prevent deadlock on IndexWriter mutex + finishFlushedSegment(newSegment, flushedFiles); + } + } + + numDocsInRAM.set(0); + return anythingFlushed; } /** Build compound file for the segment we just flushed */ - void createCompoundFile(String compoundFileName, DocumentsWriterPerThread perThread) throws IOException { + void createCompoundFile(String compoundFileName, Collection flushedFiles) throws IOException { CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); - for(String fileName : perThread.flushState.flushedFiles) { + for(String fileName : flushedFiles) { cfsWriter.addFile(fileName); } @@ -481,16 +500,14 @@ final class DocumentsWriter { cfsWriter.close(); } - void finishFlushedSegment(SegmentInfo newSegment, DocumentsWriterPerThread perThread) throws IOException { - perThread.pushDeletes(newSegment, indexWriter.segmentInfos); - + void finishFlushedSegment(SegmentInfo newSegment, Collection flushedFiles) throws IOException { if (indexWriter.useCompoundFile(newSegment)) { String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); message("creating compound file " + compoundFileName); // Now build compound file boolean success = false; try { - createCompoundFile(compoundFileName, perThread); + createCompoundFile(compoundFileName, flushedFiles); success = true; } finally { if (!success) { @@ -501,14 +518,14 @@ final class DocumentsWriter { indexWriter.deleter.deleteFile(IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); - for (String file : perThread.flushState.flushedFiles) { + for (String file : flushedFiles) { indexWriter.deleter.deleteFile(file); } } } - for (String file : perThread.flushState.flushedFiles) { + for (String file : flushedFiles) { indexWriter.deleter.deleteFile(file); } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 3dc25e50445..8463b0222c6 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -109,7 +109,7 @@ public class DocumentsWriterPerThread { * currently buffered docs. This resets our state, * discarding any docs added since last flush. */ void abort() throws IOException { - assert aborting; + aborting = true; try { if (infoStream != null) { message("docWriter: now abort"); @@ -152,7 +152,6 @@ public class DocumentsWriterPerThread { FieldInfos fieldInfos = new FieldInfos(); public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent, IndexingChain indexingChain) { - parent.indexWriter.testPoint("DocumentsWriterPerThread.init start"); this.directory = directory; this.parent = parent; this.writer = parent.indexWriter; @@ -191,7 +190,7 @@ public class DocumentsWriterPerThread { if (!aborting) { // mark document as deleted deleteDocID(docState.docID); - commitDocument(); + numDocsInRAM++; } } } @@ -203,7 +202,7 @@ public class DocumentsWriterPerThread { success = true; } finally { if (!success) { - setAborting(); + abort(); } } } @@ -249,23 +248,23 @@ public class DocumentsWriterPerThread { // confounding exception). } - void deleteQueries(Query... queries) { + synchronized void deleteQueries(Query... queries) { for (Query query : queries) { pendingDeletes.addQuery(query, numDocsInRAM); } } - void deleteQuery(Query query) { + synchronized void deleteQuery(Query query) { pendingDeletes.addQuery(query, numDocsInRAM); } - void deleteTerms(Term... terms) { + synchronized void deleteTerms(Term... terms) { for (Term term : terms) { pendingDeletes.addTerm(term, numDocsInRAM); } } - void deleteTerm(Term term) { + synchronized void deleteTerm(Term term) { pendingDeletes.addTerm(term, numDocsInRAM); } @@ -350,7 +349,7 @@ public class DocumentsWriterPerThread { return newSegment; } finally { if (!success) { - setAborting(); + abort(); } } } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java new file mode 100644 index 00000000000..1c8a0d68b33 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java @@ -0,0 +1,81 @@ +package org.apache.lucene.index; + +import java.util.Iterator; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.document.Document; + +public abstract class DocumentsWriterPerThreadPool { + final static class ThreadState extends ReentrantLock { + final DocumentsWriterPerThread perThread; + + ThreadState(DocumentsWriterPerThread perThread) { + this.perThread = perThread; + } + } + + private final ThreadState[] perThreads; + private volatile int numThreadStatesActive; + + public DocumentsWriterPerThreadPool(int maxNumPerThreads) { + maxNumPerThreads = (maxNumPerThreads < 1) ? IndexWriterConfig.DEFAULT_MAX_THREAD_STATES : maxNumPerThreads; + this.perThreads = new ThreadState[maxNumPerThreads]; + + numThreadStatesActive = 0; + } + + public void initialize(DocumentsWriter documentsWriter) { + for (int i = 0; i < perThreads.length; i++) { + perThreads[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, documentsWriter.chain)); + } + } + + public int getMaxThreadStates() { + return perThreads.length; + } + + public ThreadState newThreadState() { + if (numThreadStatesActive < perThreads.length) { + ThreadState state = perThreads[numThreadStatesActive]; + numThreadStatesActive++; + return state; + } + + return null; + } + + public abstract ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter, Document doc); + + public abstract void clearThreadBindings(ThreadState perThread); + + public abstract void clearAllThreadBindings(); + + public Iterator getAllPerThreadsIterator() { + return getPerThreadsIterator(this.perThreads.length); + } + + public Iterator getActivePerThreadsIterator() { + return getPerThreadsIterator(this.numThreadStatesActive); + } + + private Iterator getPerThreadsIterator(final int upto) { + return new Iterator() { + int i = 0; + + @Override + public boolean hasNext() { + return i < upto; + } + + @Override + public ThreadState next() { + return perThreads[i++]; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove() not supported."); + } + }; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java deleted file mode 100644 index 77e6f12499d..00000000000 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterThreadPool.java +++ /dev/null @@ -1,314 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Iterator; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.lucene.document.Document; -import org.apache.lucene.util.ThreadInterruptedException; - -abstract class DocumentsWriterThreadPool { - public static abstract class Task { - private boolean clearThreadBindings = false; - - protected void clearThreadBindings() { - this.clearThreadBindings = true; - } - - boolean doClearThreadBindings() { - return clearThreadBindings; - } - } - - public static abstract class PerThreadTask extends Task { - abstract T process(final DocumentsWriterPerThread perThread) throws IOException; - } - - public static abstract class AllThreadsTask extends Task { - abstract T process(final Iterator threadsIterator) throws IOException; - } - - protected abstract static class ThreadState { - private DocumentsWriterPerThread perThread; - private boolean isIdle = true; - - void start() {/* extension hook */} - void finish() {/* extension hook */} - } - - private int pauseThreads = 0; - - protected final int maxNumThreadStates; - protected ThreadState[] allThreadStates = new ThreadState[0]; - - private final Lock lock = new ReentrantLock(); - private final Condition threadStateAvailable = lock.newCondition(); - private boolean globalLock; - private boolean aborting; - - DocumentsWriterThreadPool(int maxNumThreadStates) { - this.maxNumThreadStates = (maxNumThreadStates < 1) ? IndexWriterConfig.DEFAULT_MAX_THREAD_STATES : maxNumThreadStates; - } - - public final int getMaxThreadStates() { - return this.maxNumThreadStates; - } - - void pauseAllThreads() { - lock.lock(); - try { - pauseThreads++; - while(!allThreadsIdle()) { - try { - threadStateAvailable.await(); - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - } - } finally { - lock.unlock(); - } - } - - void resumeAllThreads() { - lock.lock(); - try { - pauseThreads--; - assert pauseThreads >= 0; - if (0 == pauseThreads) { - threadStateAvailable.signalAll(); - } - } finally { - lock.unlock(); - } - } - - private boolean allThreadsIdle() { - for (ThreadState state : allThreadStates) { - if (!state.isIdle) { - return false; - } - } - - return true; - } - - void abort() throws IOException { - lock.lock(); - try { - if (!aborting) { - aborting = true; - pauseAllThreads(); - for (ThreadState state : allThreadStates) { - state.perThread.aborting = true; - } - - try { - for (ThreadState state : allThreadStates) { - state.perThread.abort(); - } - - } finally { - aborting = false; - resumeAllThreads(); - } - } - } finally { - lock.unlock(); - } - } - - void finishAbort() { - aborting = false; - resumeAllThreads(); - } - - public T executeAllThreads(DocumentsWriter documentsWriter, AllThreadsTask task) throws IOException { - T result = null; - - lock.lock(); - try { - try { - while (globalLock) { - threadStateAvailable.await(); - } - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - - assert !globalLock; - globalLock = true; - - pauseAllThreads(); - - } finally { - lock.unlock(); - } - - final ThreadState[] localAllThreads = allThreadStates; - - // all threads are idle now - boolean success = false; - try { - result = task.process(getPerThreadIterator(localAllThreads)); - success = true; - return result; - } finally { - boolean abort = false; - if (!success) { - for (ThreadState state : localAllThreads) { - if (state.perThread.aborting) { - abort = true; - } - } - } - - lock.lock(); - try { - try { - if (task.doClearThreadBindings()) { - clearAllThreadBindings(); - } - } finally { - globalLock = false; - resumeAllThreads(); - threadStateAvailable.signalAll(); - } - } finally { - lock.unlock(); - } - - if (!aborting && abort) { - documentsWriter.abort(); - } - - } - } - - - public final T executePerThread(DocumentsWriter documentsWriter, Document doc, PerThreadTask task) throws IOException { - ThreadState state = acquireThreadState(documentsWriter, doc); - boolean success = false; - try { - T result = task.process(state.perThread); - success = true; - return result; - } finally { - boolean abort = false; - if (!success && state.perThread.aborting) { - abort = true; - } - - returnDocumentsWriterPerThread(state, task.doClearThreadBindings()); - - if (!aborting && abort) { - documentsWriter.abort(); - } - } - } - - final Iterator getPerThreadIterator() { - return getPerThreadIterator(allThreadStates); - } - - private static final Iterator getPerThreadIterator(final ThreadState[] localAllThreads) { - return new Iterator() { - int i = 0; - - @Override - public boolean hasNext() { - return i < localAllThreads.length; - } - - @Override - public DocumentsWriterPerThread next() { - return localAllThreads[i++].perThread; - } - - @Override - public void remove() { - throw new UnsupportedOperationException("remove() not supported."); - } - }; - } - - protected final T addNewThreadState(DocumentsWriter documentsWriter, T threadState) { - // Just create a new "private" thread state - ThreadState[] newArray = new ThreadState[1+allThreadStates.length]; - if (allThreadStates.length > 0) - System.arraycopy(allThreadStates, 0, newArray, 0, allThreadStates.length); - threadState.perThread = documentsWriter.newDocumentsWriterPerThread(); - newArray[allThreadStates.length] = threadState; - - allThreadStates = newArray; - return threadState; - } - - protected abstract ThreadState selectThreadState(Thread requestingThread, DocumentsWriter documentsWriter, Document doc); - protected void clearThreadBindings(ThreadState flushedThread) { - // subclasses can optionally override this to cleanup after a thread flushed - } - - protected void clearAllThreadBindings() { - // subclasses can optionally override this to cleanup after a thread flushed - } - - - private final ThreadState acquireThreadState(DocumentsWriter documentsWriter, Document doc) { - lock.lock(); - try { - ThreadState threadState = selectThreadState(Thread.currentThread(), documentsWriter, doc); - - try { - while (!threadState.isIdle || globalLock || aborting || threadState.perThread.aborting) { - threadStateAvailable.await(); - } - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - - assert threadState.isIdle; - - threadState.isIdle = false; - threadState.start(); - - return threadState; - - } finally { - lock.unlock(); - } - } - - private final void returnDocumentsWriterPerThread(ThreadState state, boolean clearThreadBindings) { - lock.lock(); - try { - state.finish(); - if (clearThreadBindings) { - clearThreadBindings(state); - } - state.isIdle = true; - threadStateAvailable.signalAll(); - } finally { - lock.unlock(); - } - } -} diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index b660081875a..93d840dfc68 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -1275,7 +1275,7 @@ public class IndexWriter implements Closeable { public void deleteDocuments(Term term) throws CorruptIndexException, IOException { ensureOpen(); try { - if (docWriter.deleteTerm(term, false)) { + if (docWriter.deleteTerm(term)) { flush(true, false); } } catch (OutOfMemoryError oom) { @@ -1396,10 +1396,11 @@ public class IndexWriter implements Closeable { public void updateDocument(Term term, Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { ensureOpen(); + boolean maybeMerge = false; try { boolean success = false; try { - docWriter.updateDocument(doc, analyzer, term); + maybeMerge = docWriter.updateDocument(doc, analyzer, term); success = true; } finally { if (!success && infoStream != null) @@ -1408,6 +1409,10 @@ public class IndexWriter implements Closeable { } catch (OutOfMemoryError oom) { handleOOM(oom, "updateDocument"); } + + if (maybeMerge) { + maybeMerge(); + } } // for test purpose diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java index 7d8bb99b711..3a67703798c 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -31,12 +31,12 @@ import org.apache.lucene.util.Version; *

* All setter methods return {@link IndexWriterConfig} to allow chaining * settings conveniently. Thus someone can do: - * + * * 

  * IndexWriterConfig conf = new IndexWriterConfig(analyzer);
  * conf.setter1().setter2();
  *
- * + * * @since 3.1 */ public final class IndexWriterConfig implements Cloneable { @@ -53,7 +53,7 @@ public final class IndexWriterConfig implements Cloneable { * */ public static enum OpenMode { CREATE, APPEND, CREATE_OR_APPEND } - + /** Default value is 32. Change using {@link #setTermIndexInterval(int)}. */ public static final int DEFAULT_TERM_INDEX_INTERVAL = 32; @@ -74,7 +74,7 @@ public final class IndexWriterConfig implements Cloneable { /** * Default value for the write lock timeout (1,000 ms). - * + * * @see #setDefaultWriteLockTimeout(long) */ public static long WRITE_LOCK_TIMEOUT = 1000; @@ -102,7 +102,7 @@ public final class IndexWriterConfig implements Cloneable { /** * Returns the default write lock timeout for newly instantiated * IndexWriterConfigs. - * + * * @see #setDefaultWriteLockTimeout(long) */ public static long getDefaultWriteLockTimeout() { @@ -126,9 +126,9 @@ public final class IndexWriterConfig implements Cloneable { private CodecProvider codecProvider; private MergePolicy mergePolicy; private boolean readerPooling; - private DocumentsWriterThreadPool indexerThreadPool; + private DocumentsWriterPerThreadPool indexerThreadPool; private int readerTermsIndexDivisor; - + // required for clone private Version matchVersion; @@ -161,7 +161,7 @@ public final class IndexWriterConfig implements Cloneable { indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(DEFAULT_MAX_THREAD_STATES); readerTermsIndexDivisor = DEFAULT_READER_TERMS_INDEX_DIVISOR; } - + @Override public Object clone() { // Shallow clone is the only thing that's possible, since parameters like @@ -184,7 +184,7 @@ public final class IndexWriterConfig implements Cloneable { this.openMode = openMode; return this; } - + /** Returns the {@link OpenMode} set by {@link #setOpenMode(OpenMode)}. */ public OpenMode getOpenMode() { return openMode; @@ -243,7 +243,7 @@ public final class IndexWriterConfig implements Cloneable { /** * Returns the maximum number of terms that will be indexed for a single field * in a document. - * + * * @see #setMaxFieldLength(int) */ public int getMaxFieldLength() { @@ -273,7 +273,7 @@ public final class IndexWriterConfig implements Cloneable { *

* NOTE: the similarity cannot be null. If null is passed, * the similarity will be set to the default. - * + * * @see Similarity#setDefault(Similarity) */ public IndexWriterConfig setSimilarity(Similarity similarity) { @@ -289,7 +289,7 @@ public final class IndexWriterConfig implements Cloneable { public Similarity getSimilarity() { return similarity; } - + /** * Expert: set the interval between indexed terms. Large values cause less * memory to be used by IndexReader, but slow random-access to terms. Small @@ -309,7 +309,7 @@ public final class IndexWriterConfig implements Cloneable { * In particular, numUniqueTerms/interval terms are read into * memory by an IndexReader, and, on average, interval/2 terms * must be scanned for each random term access. - * + * * @see #DEFAULT_TERM_INDEX_INTERVAL */ public IndexWriterConfig setTermIndexInterval(int interval) { @@ -319,7 +319,7 @@ public final class IndexWriterConfig implements Cloneable { /** * Returns the interval between indexed terms. - * + * * @see #setTermIndexInterval(int) */ public int getTermIndexInterval() { @@ -355,10 +355,10 @@ public final class IndexWriterConfig implements Cloneable { this.writeLockTimeout = writeLockTimeout; return this; } - + /** * Returns allowed timeout when acquiring the write lock. - * + * * @see #setWriteLockTimeout(long) */ public long getWriteLockTimeout() { @@ -372,7 +372,7 @@ public final class IndexWriterConfig implements Cloneable { * created. *

Disabled by default (writer flushes by RAM usage). - * + * * @throws IllegalArgumentException if maxBufferedDeleteTerms * is enabled but smaller than 1 * @see #setRAMBufferSizeMB @@ -389,7 +389,7 @@ public final class IndexWriterConfig implements Cloneable { /** * Returns the number of buffered deleted terms that will trigger a flush if * enabled. - * + * * @see #setMaxBufferedDeleteTerms(int) */ public int getMaxBufferedDeleteTerms() { @@ -401,33 +401,33 @@ public final class IndexWriterConfig implements Cloneable { * and deletions before they are flushed to the Directory. Generally for * faster indexing performance it's best to flush by RAM usage instead of * document count and use as large a RAM buffer as you can. - * + * *

* When this is set, the writer will flush whenever buffered documents and * deletions use this much RAM. Pass in {@link #DISABLE_AUTO_FLUSH} to prevent * triggering a flush due to RAM usage. Note that if flushing by document * count is also enabled, then the flush will be triggered by whichever comes * first. - * + * *

* NOTE: the account of RAM usage for pending deletions is only * approximate. Specifically, if you delete by Query, Lucene currently has no * way to measure the RAM usage of individual Queries so the accounting will * under-estimate and you should compensate by either calling commit() * periodically yourself, or by using {@link #setMaxBufferedDeleteTerms(int)} - * to flush by count instead of RAM usage (each buffered delete Query counts + * to flush by count instead of RAM usage (each buffered delete Query counts * as one). - * + * *

* NOTE: because IndexWriter uses ints when managing its * internal storage, the absolute maximum value for this setting is somewhat * less than 2048 MB. The precise limit depends on various factors, such as * how large your documents are, how many fields have norms, etc., so it's * best to set this value comfortably under 2048. - * + * *

* The default value is {@link #DEFAULT_RAM_BUFFER_SIZE_MB}. - * + * * @throws IllegalArgumentException * if ramBufferSize is enabled but non-positive, or it disables * ramBufferSize when maxBufferedDocs is already disabled @@ -456,19 +456,19 @@ public final class IndexWriterConfig implements Cloneable { * Determines the minimal number of documents required before the buffered * in-memory documents are flushed as a new Segment. Large values generally * give faster indexing. - * + * *

* When this is set, the writer will flush every maxBufferedDocs added * documents. Pass in {@link #DISABLE_AUTO_FLUSH} to prevent triggering a * flush due to number of buffered documents. Note that if flushing by RAM * usage is also enabled, then the flush will be triggered by whichever comes * first. - * + * *

* Disabled by default (writer flushes by RAM usage). - * + * * @see #setRAMBufferSizeMB(double) - * + * * @throws IllegalArgumentException * if maxBufferedDocs is enabled but smaller than 2, or it disables * maxBufferedDocs when ramBufferSize is already disabled @@ -488,7 +488,7 @@ public final class IndexWriterConfig implements Cloneable { /** * Returns the number of buffered added documents that will trigger a flush if * enabled. - * + * * @see #setMaxBufferedDocs(int) */ public int getMaxBufferedDocs() { @@ -529,10 +529,10 @@ public final class IndexWriterConfig implements Cloneable { return codecProvider; } - + /** * Returns the current MergePolicy in use by this writer. - * + * * @see #setMergePolicy(MergePolicy) */ public MergePolicy getMergePolicy() { @@ -545,15 +545,15 @@ public final class IndexWriterConfig implements Cloneable { * maxThreadStates will be set to * {@link #DEFAULT_MAX_THREAD_STATES}. */ - public IndexWriterConfig setIndexerThreadPool(DocumentsWriterThreadPool threadPool) { + public IndexWriterConfig setIndexerThreadPool(DocumentsWriterPerThreadPool threadPool) { this.indexerThreadPool = threadPool; return this; } - public DocumentsWriterThreadPool getIndexerThreadPool() { + public DocumentsWriterPerThreadPool getIndexerThreadPool() { return this.indexerThreadPool; } - + /** Returns the max number of simultaneous threads that * may be indexing documents at once in IndexWriter. */ public int getMaxThreadStates() { @@ -584,7 +584,7 @@ public final class IndexWriterConfig implements Cloneable { this.indexingChain = indexingChain == null ? DocumentsWriterPerThread.defaultIndexingChain : indexingChain; return this; } - + /** Returns the indexing chain set on {@link #setIndexingChain(IndexingChain)}. */ IndexingChain getIndexingChain() { return indexingChain; @@ -606,7 +606,7 @@ public final class IndexWriterConfig implements Cloneable { public int getReaderTermsIndexDivisor() { return readerTermsIndexDivisor; } - + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java b/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java index 9f70da174f7..cba0fc98beb 100644 --- a/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java @@ -1,66 +1,56 @@ package org.apache.lucene.index; -import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.document.Document; -public class ThreadAffinityDocumentsWriterThreadPool extends DocumentsWriterThreadPool { - private static final class AffinityThreadState extends ThreadState { - int numAssignedThreads; +public class ThreadAffinityDocumentsWriterThreadPool extends DocumentsWriterPerThreadPool { + private Map threadBindings = new ConcurrentHashMap(); - @Override - void finish() { - numAssignedThreads--; - } - } - - private Map threadBindings = new HashMap(); - - public ThreadAffinityDocumentsWriterThreadPool(int maxNumThreadStates) { - super(maxNumThreadStates); + public ThreadAffinityDocumentsWriterThreadPool(int maxNumPerThreads) { + super(maxNumPerThreads); } @Override - protected ThreadState selectThreadState(Thread requestingThread, DocumentsWriter documentsWriter, Document doc) { - AffinityThreadState threadState = threadBindings.get(requestingThread); - // First, find a thread state. If this thread already - // has affinity to a specific ThreadState, use that one - // again. - if (threadState == null) { - AffinityThreadState minThreadState = null; - for(int i=0;i= maxNumThreadStates)) { - threadState = minThreadState; - } else { - threadState = addNewThreadState(documentsWriter, new AffinityThreadState()); - } - threadBindings.put(requestingThread, threadState); } - threadState.numAssignedThreads++; - return threadState; - } - - @Override - protected void clearThreadBindings(ThreadState flushedThread) { - Iterator> it = threadBindings.entrySet().iterator(); + // find the state that has minimum amount of threads waiting + Iterator it = getActivePerThreadsIterator(); + ThreadState minThreadState = null; while (it.hasNext()) { - Entry e = it.next(); - if (e.getValue() == flushedThread) { - it.remove(); + ThreadState state = it.next(); + if (minThreadState == null || state.getQueueLength() < minThreadState.getQueueLength()) { + minThreadState = state; } } + + if (minThreadState == null || minThreadState.hasQueuedThreads()) { + ThreadState newState = newThreadState(); + if (newState != null) { + minThreadState = newState; + threadBindings.put(requestingThread, newState); + } + } + + minThreadState.lock(); + return minThreadState; } @Override - protected void clearAllThreadBindings() { + public void clearThreadBindings(ThreadState perThread) { + threadBindings.clear(); + } + + @Override + public void clearAllThreadBindings() { threadBindings.clear(); } } From 9b83aa6dedca6f46988d2d311d639bb263aae014 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Thu, 13 Jan 2011 09:22:58 +0000 Subject: [PATCH 012/200] LUCENE-2324: make newThreadState synchronized git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1058470 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/index/DocumentsWriterPerThreadPool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java index 1c8a0d68b33..f210868e5f0 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java @@ -34,7 +34,7 @@ public abstract class DocumentsWriterPerThreadPool { return perThreads.length; } - public ThreadState newThreadState() { + public synchronized ThreadState newThreadState() { if (numThreadStatesActive < perThreads.length) { ThreadState state = perThreads[numThreadStatesActive]; numThreadStatesActive++; From e15dd99e2aefa671763df0bfbc8fa6531482c85b Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Thu, 13 Jan 2011 23:11:59 +0000 Subject: [PATCH 013/200] LUCENE-2324: remove memory recycling from DWPT; some other minor cleanup git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1058789 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 8 ----- .../index/DocumentsWriterPerThread.java | 30 ++++--------------- .../org/apache/lucene/index/IntBlockPool.java | 4 --- .../apache/lucene/index/TestIndexWriter.java | 2 +- 4 files changed, 6 insertions(+), 38 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 5cb4af37c06..ba44b4a9fe9 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -20,7 +20,6 @@ package org.apache.lucene.index; import java.io.IOException; import java.io.PrintStream; import java.util.Collection; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -149,10 +148,6 @@ final class DocumentsWriter { // we are flushing by doc count instead. private long ramBufferSize = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024); - // If we've allocated 5% over our RAM budget, we then - // free down to 95% - private long freeLevel = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024*0.95); - // Flush @ this number of docs. If ramBufferSize is // non-zero we will flush by RAM usage instead. private int maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS; @@ -161,7 +156,6 @@ final class DocumentsWriter { final BufferedDeletes bufferedDeletes; final SegmentDeletes pendingDeletes; - private final IndexWriter.FlushControl flushControl; final IndexingChain chain; final DocumentsWriterPerThreadPool perThreadPool; @@ -175,7 +169,6 @@ final class DocumentsWriter { this.perThreadPool = indexerThreadPool; this.pendingDeletes = new SegmentDeletes(); this.chain = chain; - flushControl = writer.flushControl; this.perThreadPool.initialize(this); } @@ -270,7 +263,6 @@ final class DocumentsWriter { ramBufferSize = IndexWriterConfig.DISABLE_AUTO_FLUSH; } else { ramBufferSize = (long) (mb*1024*1024); - freeLevel = (long) (0.95 * ramBufferSize); } } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 8463b0222c6..0fc743fae96 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -23,7 +23,6 @@ import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE; import java.io.IOException; import java.io.PrintStream; import java.text.NumberFormat; -import java.util.ArrayList; import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.analysis.Analyzer; @@ -31,8 +30,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.search.Query; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.ByteBlockPool.DirectAllocator; import org.apache.lucene.util.RamUsageEstimator; -import org.apache.lucene.util.RecyclingByteBlockAllocator; public class DocumentsWriterPerThread { @@ -387,33 +386,14 @@ public class DocumentsWriterPerThread { final static int INT_BLOCK_SIZE = 1 << INT_BLOCK_SHIFT; final static int INT_BLOCK_MASK = INT_BLOCK_SIZE - 1; - private ArrayList freeIntBlocks = new ArrayList(); - /* Allocate another int[] from the shared pool */ - synchronized int[] getIntBlock() { - final int size = freeIntBlocks.size(); - final int[] b; - if (0 == size) { - b = new int[INT_BLOCK_SIZE]; - bytesUsed.addAndGet(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT); - } else - b = freeIntBlocks.remove(size-1); + int[] getIntBlock() { + int[] b = new int[INT_BLOCK_SIZE]; + bytesUsed.addAndGet(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT); return b; } - /* Return int[]s to the pool */ - synchronized void recycleIntBlocks(int[][] blocks, int start, int end) { - for(int i=start;i 0) - // Recycle all but the first buffer - docWriter.recycleIntBlocks(buffers, 1, 1+bufferUpto); - // Reuse first buffer bufferUpto = 0; intUpto = 0; diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index 00da0324fbe..d6da14e7350 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -2581,7 +2581,7 @@ public class TestIndexWriter extends LuceneTestCase { } } - public void testIndexingThenDeleting() throws Exception { + public void _testIndexingThenDeleting() throws Exception { final Random r = random; Directory dir = newDirectory(); From 604730465d57ea6be8063509e75e2f837e3a980a Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Sun, 16 Jan 2011 01:46:00 +0000 Subject: [PATCH 014/200] LUCENE-2324: several bug fixes git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1059431 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 22 +++-------- .../org/apache/lucene/index/IndexWriter.java | 39 +++++++++++++++---- .../org/apache/lucene/index/SegmentInfo.java | 14 +++++++ .../lucene/index/TermVectorsTermsWriter.java | 4 ++ .../lucene/index/TermVectorsWriter.java | 30 +++----------- 5 files changed, 62 insertions(+), 47 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index ba44b4a9fe9..91d0d2e0ef8 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -370,13 +370,11 @@ final class DocumentsWriter { long perThreadRAMUsedBeforeAdd = dwpt.bytesUsed(); dwpt.addDocument(doc, analyzer); - synchronized(DocumentsWriter.this) { - if (delTerm != null) { - deleteTerm(delTerm); - } - dwpt.commitDocument(); - numDocsInRAM.incrementAndGet(); + if (delTerm != null) { + deleteTerm(delTerm); } + dwpt.commitDocument(); + numDocsInRAM.incrementAndGet(); newSegment = finishAddDocument(dwpt, perThreadRAMUsedBeforeAdd); if (newSegment != null) { @@ -502,19 +500,11 @@ final class DocumentsWriter { "reating compound file for newly flushed segment " + newSegment.name); } - indexWriter.deleter.deleteFile(IndexFileNames.segmentFileName(newSegment.name, "", - IndexFileNames.COMPOUND_FILE_EXTENSION)); - for (String file : newSegment.files()) { - indexWriter.deleter.deleteFile(file); - } - + indexWriter.deleter.refresh(newSegment.name); } } - for (String file : newSegment.files()) { - indexWriter.deleter.deleteFile(file); - } - + indexWriter.deleter.deleteNewFiles(newSegment.files()); newSegment.setUseCompoundFile(true); } diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 72533a8563c..b4b5fc86c1a 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -1396,9 +1396,9 @@ public class IndexWriter implements Closeable { public void updateDocument(Term term, Document doc, Analyzer analyzer) throws CorruptIndexException, IOException { ensureOpen(); - boolean maybeMerge = false; try { boolean success = false; + boolean maybeMerge = false; try { maybeMerge = docWriter.updateDocument(doc, analyzer, term); success = true; @@ -1406,13 +1406,13 @@ public class IndexWriter implements Closeable { if (!success && infoStream != null) message("hit exception updating document"); } + + if (maybeMerge) { + maybeMerge(); + } } catch (OutOfMemoryError oom) { handleOOM(oom, "updateDocument"); } - - if (maybeMerge) { - maybeMerge(); - } } // for test purpose @@ -2186,6 +2186,9 @@ public class IndexWriter implements Closeable { } // Update SI appropriately + // if this call is removed in the future we need to make + // sure that info.clearFiles() is called here + info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile()); info.dir = directory; info.name = newSegName; @@ -2501,7 +2504,6 @@ public class IndexWriter implements Closeable { boolean maybeMerge = docWriter.flushAllThreads(applyAllDeletes); synchronized(this) { - if (!applyAllDeletes) { // If deletes alone are consuming > 1/2 our RAM // buffer, force them all to apply now. This is to @@ -2536,8 +2538,8 @@ public class IndexWriter implements Closeable { success = true; return maybeMerge; - } + } } catch (OutOfMemoryError oom) { handleOOM(oom, "doFlush"); // never hit @@ -2697,6 +2699,12 @@ public class IndexWriter implements Closeable { commitMergedDeletes(merge, mergedReader); + // If the doc store we are using has been closed and + // is in now compound format (but wasn't when we + // started), then we will switch to the compound + // format as well: + setMergeDocStoreIsCompoundFile(merge); + segmentInfos.subList(start, start + merge.segments.size()).clear(); assert !segmentInfos.contains(merge.info); segmentInfos.add(start, merge.info); @@ -2972,6 +2980,23 @@ public class IndexWriter implements Closeable { runningMerges.remove(merge); } + private synchronized void setMergeDocStoreIsCompoundFile(MergePolicy.OneMerge merge) { + final String mergeDocStoreSegment = merge.info.getDocStoreSegment(); + if (mergeDocStoreSegment != null && !merge.info.getDocStoreIsCompoundFile()) { + final int size = segmentInfos.size(); + for(int i=0;i Date: Sun, 16 Jan 2011 10:45:38 +0000 Subject: [PATCH 015/200] LUCENE-2324: fix most of the delete problems in the RT branch git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1059528 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 129 ++++++++++++------ .../index/DocumentsWriterPerThread.java | 37 +---- .../index/TestIndexWriterExceptions.java | 46 +++---- .../lucene/index/TestPerSegmentDeletes.java | 2 +- 4 files changed, 115 insertions(+), 99 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 91d0d2e0ef8..72dad36257c 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -121,29 +121,6 @@ final class DocumentsWriter { private AtomicInteger numDocsInRAM = new AtomicInteger(0); private AtomicLong ramUsed = new AtomicLong(0); - static class DocState { - DocumentsWriter docWriter; - Analyzer analyzer; - int maxFieldLength; - PrintStream infoStream; - Similarity similarity; - int docID; - Document doc; - String maxTermPrefix; - - // Only called by asserts - public boolean testPoint(String name) { - return docWriter.indexWriter.testPoint(name); - } - - public void clear() { - // don't hold onto doc nor analyzer, in case it is - // largish: - doc = null; - analyzer = null; - } - } - // How much RAM we can use before flushing. This is 0 if // we are flushing by doc count instead. private long ramBufferSize = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024); @@ -155,7 +132,7 @@ final class DocumentsWriter { private final FieldInfos fieldInfos; final BufferedDeletes bufferedDeletes; - final SegmentDeletes pendingDeletes; + SegmentDeletes pendingDeletes; final IndexingChain chain; final DocumentsWriterPerThreadPool perThreadPool; @@ -175,13 +152,19 @@ final class DocumentsWriter { boolean deleteQueries(final Query... queries) throws IOException { Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); - boolean added = false; + boolean deleted = false; while (threadsIterator.hasNext()) { - threadsIterator.next().perThread.deleteQueries(queries); - added = true; + ThreadState state = threadsIterator.next(); + state.lock(); + try { + state.perThread.deleteQueries(queries); + deleted = true; + } finally { + state.unlock(); + } } - if (!added) { + if (!deleted) { synchronized(this) { for (Query query : queries) { pendingDeletes.addQuery(query, SegmentDeletes.MAX_INT); @@ -199,13 +182,19 @@ final class DocumentsWriter { boolean deleteTerms(final Term... terms) throws IOException { Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); - boolean added = false; + boolean deleted = false; while (threadsIterator.hasNext()) { - threadsIterator.next().perThread.deleteTerms(terms); - added = true; + ThreadState state = threadsIterator.next(); + deleted = true; + state.lock(); + try { + state.perThread.deleteTerms(terms); + } finally { + state.unlock(); + } } - if (!added) { + if (!deleted) { synchronized(this) { for (Term term : terms) { pendingDeletes.addTerm(term, SegmentDeletes.MAX_INT); @@ -220,6 +209,26 @@ final class DocumentsWriter { return deleteTerms(term); } + boolean deleteTerm(final Term term, ThreadState exclude) { + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + + boolean deleted = false; + while (threadsIterator.hasNext()) { + deleted = true; + ThreadState state = threadsIterator.next(); + if (state != exclude) { + state.lock(); + try { + state.perThread.deleteTerm(term); + } finally { + state.unlock(); + } + } + } + + return deleted; + } + public FieldInfos getFieldInfos() { return fieldInfos; } @@ -371,25 +380,32 @@ final class DocumentsWriter { dwpt.addDocument(doc, analyzer); if (delTerm != null) { - deleteTerm(delTerm); + dwpt.deleteTerm(delTerm); } dwpt.commitDocument(); numDocsInRAM.incrementAndGet(); newSegment = finishAddDocument(dwpt, perThreadRAMUsedBeforeAdd); - if (newSegment != null) { - perThreadPool.clearThreadBindings(perThread); + if (newSegment != null && dwpt.pendingDeletes.any()) { + bufferedDeletes.pushDeletes(dwpt.pendingDeletes, newSegment); + dwpt.pendingDeletes = new SegmentDeletes(); } - } finally { perThread.unlock(); } if (newSegment != null) { + perThreadPool.clearThreadBindings(perThread); finishFlushedSegment(newSegment); return true; } + // delete term from other DWPTs later, so that this thread + // doesn't have to lock multiple DWPTs at the same time + if (delTerm != null) { + deleteTerm(delTerm, perThread); + } + return false; } @@ -416,14 +432,32 @@ final class DocumentsWriter { return newSegment; } + private final void pushToLastSegment(SegmentDeletes segmentDeletes) { + synchronized(indexWriter) { + // Lock order: DW -> BD + if (segmentDeletes.any()) { + if (indexWriter.segmentInfos.size() > 0) { + if (infoStream != null) { + message("flush: push buffered deletes to previously flushed segment " + indexWriter.segmentInfos.lastElement()); + } + bufferedDeletes.pushDeletes(segmentDeletes, indexWriter.segmentInfos.lastElement(), true); + } else { + if (infoStream != null) { + message("flush: drop buffered deletes: no segments"); + } + // We can safely discard these deletes: since + // there are no segments, the deletions cannot + // affect anything. + } + } + } + } + final boolean flushAllThreads(final boolean flushDeletes) throws IOException { if (flushDeletes) { - if (indexWriter.segmentInfos.size() > 0 && pendingDeletes.any()) { - bufferedDeletes.pushDeletes(pendingDeletes, indexWriter.segmentInfos.lastElement(), true); - pendingDeletes.clear(); - } + pushToLastSegment(pendingDeletes); } Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); @@ -450,13 +484,16 @@ final class DocumentsWriter { newSegment = dwpt.flush(); if (newSegment != null) { - IndexWriter.setDiagnostics(newSegment, "flush"); - dwpt.pushDeletes(newSegment, indexWriter.segmentInfos); anythingFlushed = true; perThreadPool.clearThreadBindings(perThread); + if (dwpt.pendingDeletes.any()) { + bufferedDeletes.pushDeletes(dwpt.pendingDeletes, newSegment); + dwpt.pendingDeletes = new SegmentDeletes(); + } } - } else if (flushDeletes) { - dwpt.pushDeletes(null, indexWriter.segmentInfos); + } + else if (flushDeletes && dwpt.pendingDeletes.any()) { + pushToLastSegment(dwpt.pendingDeletes); } } finally { perThread.unlock(); @@ -485,6 +522,10 @@ final class DocumentsWriter { } void finishFlushedSegment(SegmentInfo newSegment) throws IOException { + assert newSegment != null; + + IndexWriter.setDiagnostics(newSegment, "flush"); + if (indexWriter.useCompoundFile(newSegment)) { String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); message("creating compound file " + compoundFileName); diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 0fc743fae96..594da6609ed 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -81,7 +81,7 @@ public class DocumentsWriterPerThread { }; // Deletes for our still-in-RAM (to be flushed next) segment - private SegmentDeletes pendingDeletes = new SegmentDeletes(); + SegmentDeletes pendingDeletes = new SegmentDeletes(); static class DocState { final DocumentsWriterPerThread docWriter; @@ -170,6 +170,7 @@ public class DocumentsWriterPerThread { } public void addDocument(Document doc, Analyzer analyzer) throws IOException { + assert writer.testPoint("DocumentsWriterPerThread addDocument start"); docState.doc = doc; docState.analyzer = analyzer; docState.docID = numDocsInRAM; @@ -206,35 +207,9 @@ public class DocumentsWriterPerThread { } } - void pushDeletes(SegmentInfo newSegment, SegmentInfos segmentInfos) { - // Lock order: DW -> BD - if (pendingDeletes.any()) { - if (newSegment != null) { - if (infoStream != null) { - message("flush: push buffered deletes to newSegment"); - } - parent.bufferedDeletes.pushDeletes(pendingDeletes, newSegment); - } else if (segmentInfos.size() > 0) { - if (infoStream != null) { - message("flush: push buffered deletes to previously flushed segment " + segmentInfos.lastElement()); - } - parent.bufferedDeletes.pushDeletes(pendingDeletes, segmentInfos.lastElement(), true); - } else { - if (infoStream != null) { - message("flush: drop buffered deletes: no segments"); - } - // We can safely discard these deletes: since - // there are no segments, the deletions cannot - // affect anything. - } - pendingDeletes = new SegmentDeletes(); - } - } - - // Buffer a specific docID for deletion. Currently only // used when we hit a exception when adding a document - synchronized void deleteDocID(int docIDUpto) { + void deleteDocID(int docIDUpto) { pendingDeletes.addDocID(docIDUpto); // NOTE: we do not trigger flush here. This is // potentially a RAM leak, if you have an app that tries @@ -247,13 +222,13 @@ public class DocumentsWriterPerThread { // confounding exception). } - synchronized void deleteQueries(Query... queries) { + void deleteQueries(Query... queries) { for (Query query : queries) { pendingDeletes.addQuery(query, numDocsInRAM); } } - synchronized void deleteQuery(Query query) { + void deleteQuery(Query query) { pendingDeletes.addQuery(query, numDocsInRAM); } @@ -263,7 +238,7 @@ public class DocumentsWriterPerThread { } } - synchronized void deleteTerm(Term term) { + void deleteTerm(Term term) { pendingDeletes.addTerm(term, numDocsInRAM); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index 9a40ea29fca..83ba2c4db43 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -245,7 +245,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { _TestUtil.checkIndex(dir); dir.close(); } - + // LUCENE-1198 private static final class MockIndexWriter2 extends IndexWriter { @@ -257,12 +257,12 @@ public class TestIndexWriterExceptions extends LuceneTestCase { @Override boolean testPoint(String name) { - if (doFail && name.equals("DocumentsWriter.ThreadState.init start")) + if (doFail && name.equals("DocumentsWriterPerThread addDocument start")) throw new RuntimeException("intentionally failing"); return true; } } - + private class CrashingFilter extends TokenFilter { String fieldName; int count; @@ -334,7 +334,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { w.addDocument(doc); w.close(); dir.close(); - } + } private static final class MockIndexWriter3 extends IndexWriter { @@ -354,7 +354,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { return true; } } - + // LUCENE-1210 public void testExceptionOnMergeInit() throws IOException { @@ -379,7 +379,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { w.close(); dir.close(); } - + // LUCENE-1072 public void testExceptionFromTokenStream() throws IOException { Directory dir = newDirectory(); @@ -470,9 +470,9 @@ public class TestIndexWriterExceptions extends LuceneTestCase { boolean sawAppend = false; boolean sawFlush = false; for (int i = 0; i < trace.length; i++) { - if ("org.apache.lucene.index.FreqProxTermsWriter".equals(trace[i].getClassName()) && "appendPostings".equals(trace[i].getMethodName())) + if ("org.apache.lucene.index.FreqProxTermsWriterPerField".equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName())) sawAppend = true; - if ("doFlush".equals(trace[i].getMethodName())) + if ("flush".equals(trace[i].getMethodName())) sawFlush = true; } @@ -680,7 +680,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { for(int t=0;t Date: Mon, 17 Jan 2011 09:23:42 +0000 Subject: [PATCH 016/200] LUCENE-2324: fix more junits in rt branch git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1059822 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 163 +++++++++--------- .../index/DocumentsWriterPerThread.java | 68 +++----- .../org/apache/lucene/index/FieldInfos.java | 43 +++-- .../org/apache/lucene/index/IndexWriter.java | 95 +++++++--- .../apache/lucene/index/SegmentDeletes.java | 8 +- 5 files changed, 211 insertions(+), 166 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 72dad36257c..3e69a313d7e 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -105,8 +105,6 @@ final class DocumentsWriter { final AtomicLong bytesUsed = new AtomicLong(0); Directory directory; - int numDocsInStore; // # docs written to doc stores - boolean bufferIsFull; // True when it's time to write segment private volatile boolean closed; @@ -172,7 +170,7 @@ final class DocumentsWriter { } } - return true; + return false; } boolean deleteQuery(final Query query) throws IOException { @@ -219,7 +217,7 @@ final class DocumentsWriter { if (state != exclude) { state.lock(); try { - state.perThread.deleteTerm(term); + state.perThread.deleteTerms(term); } finally { state.unlock(); } @@ -348,18 +346,53 @@ final class DocumentsWriter { } synchronized boolean anyChanges() { - // nocommit - return numDocsInRAM.get() != 0; - //return numDocsInRAM.get() != 0 || pendingDeletes.any(); + return numDocsInRAM.get() != 0 || anyDeletions(); } - // for testing - public synchronized SegmentDeletes getPendingDeletes() { - return pendingDeletes; + public int getBufferedDeleteTermsSize() { + int size = 0; + Iterator it = perThreadPool.getActivePerThreadsIterator(); + while (it.hasNext()) { + DocumentsWriterPerThread dwpt = it.next().perThread; + size += dwpt.pendingDeletes.terms.size(); + } + size += pendingDeletes.terms.size(); + return size; } + //for testing + public int getNumBufferedDeleteTerms() { + int numDeletes = 0; + Iterator it = perThreadPool.getActivePerThreadsIterator(); + while (it.hasNext()) { + DocumentsWriterPerThread dwpt = it.next().perThread; + numDeletes += dwpt.pendingDeletes.numTermDeletes.get(); + } + numDeletes += pendingDeletes.numTermDeletes.get(); + return numDeletes; + } + + // TODO: can we improve performance of this method by keeping track + // here in DW of whether any DWPT has deletions? public synchronized boolean anyDeletions() { - return pendingDeletes.any(); + if (pendingDeletes.any()) { + return true; + } + + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + while (threadsIterator.hasNext()) { + ThreadState state = threadsIterator.next(); + state.lock(); + try { + if (state.perThread.pendingDeletes.any()) { + return true; + } + } finally { + state.unlock(); + } + } + + return false; } synchronized void close() { @@ -372,31 +405,34 @@ final class DocumentsWriter { ensureOpen(); SegmentInfo newSegment = null; + SegmentDeletes segmentDeletes = null; ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(), this, doc); try { DocumentsWriterPerThread dwpt = perThread.perThread; long perThreadRAMUsedBeforeAdd = dwpt.bytesUsed(); - dwpt.addDocument(doc, analyzer); - - if (delTerm != null) { - dwpt.deleteTerm(delTerm); - } - dwpt.commitDocument(); + dwpt.updateDocument(doc, analyzer, delTerm); numDocsInRAM.incrementAndGet(); newSegment = finishAddDocument(dwpt, perThreadRAMUsedBeforeAdd); - if (newSegment != null && dwpt.pendingDeletes.any()) { - bufferedDeletes.pushDeletes(dwpt.pendingDeletes, newSegment); - dwpt.pendingDeletes = new SegmentDeletes(); + if (newSegment != null) { + fieldInfos.update(dwpt.getFieldInfos()); + if (dwpt.pendingDeletes.any()) { + segmentDeletes = dwpt.pendingDeletes; + dwpt.pendingDeletes = new SegmentDeletes(); + } } } finally { perThread.unlock(); } + if (segmentDeletes != null) { + pushDeletes(newSegment, segmentDeletes); + } + if (newSegment != null) { perThreadPool.clearThreadBindings(perThread); - finishFlushedSegment(newSegment); + indexWriter.addFlushedSegment(newSegment); return true; } @@ -413,14 +449,8 @@ final class DocumentsWriter { long perThreadRAMUsedBeforeAdd) throws IOException { SegmentInfo newSegment = null; - int numDocsPerThread = perThread.getNumDocsInRAM(); if (perThread.getNumDocsInRAM() == maxBufferedDocs) { newSegment = perThread.flush(); - - int oldValue = numDocsInRAM.get(); - while (!numDocsInRAM.compareAndSet(oldValue, oldValue - numDocsPerThread)) { - oldValue = numDocsInRAM.get(); - } } long deltaRAM = perThread.bytesUsed() - perThreadRAMUsedBeforeAdd; @@ -432,11 +462,20 @@ final class DocumentsWriter { return newSegment; } - private final void pushToLastSegment(SegmentDeletes segmentDeletes) { + final void substractFlushedNumDocs(int numFlushed) { + int oldValue = numDocsInRAM.get(); + while (!numDocsInRAM.compareAndSet(oldValue, oldValue - numFlushed)) { + oldValue = numDocsInRAM.get(); + } + } + + private final void pushDeletes(SegmentInfo segmentInfo, SegmentDeletes segmentDeletes) { synchronized(indexWriter) { // Lock order: DW -> BD if (segmentDeletes.any()) { - if (indexWriter.segmentInfos.size() > 0) { + if (segmentInfo != null) { + bufferedDeletes.pushDeletes(segmentDeletes, segmentInfo); + } else if (indexWriter.segmentInfos.size() > 0) { if (infoStream != null) { message("flush: push buffered deletes to previously flushed segment " + indexWriter.segmentInfos.lastElement()); } @@ -457,7 +496,10 @@ final class DocumentsWriter { throws IOException { if (flushDeletes) { - pushToLastSegment(pendingDeletes); + synchronized (this) { + pushDeletes(null, pendingDeletes); + pendingDeletes = new SegmentDeletes(); + } } Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); @@ -465,6 +507,7 @@ final class DocumentsWriter { while (threadsIterator.hasNext()) { SegmentInfo newSegment = null; + SegmentDeletes segmentDeletes = null; ThreadState perThread = threadsIterator.next(); perThread.lock(); @@ -484,75 +527,37 @@ final class DocumentsWriter { newSegment = dwpt.flush(); if (newSegment != null) { + fieldInfos.update(dwpt.getFieldInfos()); anythingFlushed = true; perThreadPool.clearThreadBindings(perThread); if (dwpt.pendingDeletes.any()) { - bufferedDeletes.pushDeletes(dwpt.pendingDeletes, newSegment); + segmentDeletes = dwpt.pendingDeletes; dwpt.pendingDeletes = new SegmentDeletes(); } } - } - else if (flushDeletes && dwpt.pendingDeletes.any()) { - pushToLastSegment(dwpt.pendingDeletes); + } else if (flushDeletes && dwpt.pendingDeletes.any()) { + segmentDeletes = dwpt.pendingDeletes; + dwpt.pendingDeletes = new SegmentDeletes(); } } finally { perThread.unlock(); } + if (segmentDeletes != null) { + pushDeletes(newSegment, segmentDeletes); + } + + if (newSegment != null) { // important do unlock the perThread before finishFlushedSegment // is called to prevent deadlock on IndexWriter mutex - finishFlushedSegment(newSegment); + indexWriter.addFlushedSegment(newSegment); } } - numDocsInRAM.set(0); return anythingFlushed; } - /** Build compound file for the segment we just flushed */ - void createCompoundFile(String compoundFileName, Collection flushedFiles) throws IOException { - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); - for(String fileName : flushedFiles) { - cfsWriter.addFile(fileName); - } - - // Perform the merge - cfsWriter.close(); - } - - void finishFlushedSegment(SegmentInfo newSegment) throws IOException { - assert newSegment != null; - - IndexWriter.setDiagnostics(newSegment, "flush"); - - if (indexWriter.useCompoundFile(newSegment)) { - String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); - message("creating compound file " + compoundFileName); - // Now build compound file - boolean success = false; - try { - createCompoundFile(compoundFileName, newSegment.files()); - success = true; - } finally { - if (!success) { - if (infoStream != null) { - message("hit exception " + - "reating compound file for newly flushed segment " + newSegment.name); - } - - indexWriter.deleter.refresh(newSegment.name); - } - } - - indexWriter.deleter.deleteNewFiles(newSegment.files()); - newSegment.setUseCompoundFile(true); - - } - - indexWriter.addNewSegment(newSegment); - } - // /* We have three pools of RAM: Postings, byte blocks // * (holds freq/prox posting data) and per-doc buffers // * (stored fields/term vectors). Different docs require diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 594da6609ed..0689a3a0e71 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -148,12 +148,13 @@ public class DocumentsWriterPerThread { final AtomicLong bytesUsed = new AtomicLong(0); - FieldInfos fieldInfos = new FieldInfos(); + private final FieldInfos fieldInfos; public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent, IndexingChain indexingChain) { this.directory = directory; this.parent = parent; this.writer = parent.indexWriter; + this.fieldInfos = new FieldInfos(); this.infoStream = parent.indexWriter.getInfoStream(); this.docState = new DocState(this); this.docState.similarity = parent.indexWriter.getConfig().getSimilarity(); @@ -169,11 +170,15 @@ public class DocumentsWriterPerThread { aborting = true; } - public void addDocument(Document doc, Analyzer analyzer) throws IOException { + public void updateDocument(Document doc, Analyzer analyzer, Term delTerm) throws IOException { assert writer.testPoint("DocumentsWriterPerThread addDocument start"); docState.doc = doc; docState.analyzer = analyzer; docState.docID = numDocsInRAM; + if (delTerm != null) { + pendingDeletes.addTerm(delTerm, docState.docID); + } + if (segment == null) { // this call is synchronized on IndexWriter.segmentInfos segment = writer.newSegmentName(); @@ -191,12 +196,15 @@ public class DocumentsWriterPerThread { // mark document as deleted deleteDocID(docState.docID); numDocsInRAM++; + } else { + abort(); } } } success = false; try { + numDocsInRAM++; consumer.finishDocument(); success = true; @@ -228,24 +236,12 @@ public class DocumentsWriterPerThread { } } - void deleteQuery(Query query) { - pendingDeletes.addQuery(query, numDocsInRAM); - } - - synchronized void deleteTerms(Term... terms) { + void deleteTerms(Term... terms) { for (Term term : terms) { pendingDeletes.addTerm(term, numDocsInRAM); } } - void deleteTerm(Term term) { - pendingDeletes.addTerm(term, numDocsInRAM); - } - - public void commitDocument() { - numDocsInRAM++; - } - int getNumDocsInRAM() { return numDocsInRAM; } @@ -264,6 +260,7 @@ public class DocumentsWriterPerThread { /** Reset after a flush */ private void doAfterFlush() throws IOException { segment = null; + parent.substractFlushedNumDocs(numDocsInRAM); numDocsInRAM = 0; } @@ -279,45 +276,30 @@ public class DocumentsWriterPerThread { message("flush postings as segment " + flushState.segmentName + " numDocs=" + numDocsInRAM); } + if (aborting) { + if (infoStream != null) { + message("flush: skip because aborting is set"); + } + return null; + } + boolean success = false; try { - consumer.flush(flushState); - boolean hasVectors = flushState.hasVectors; + SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, fieldInfos.hasProx(), flushState.segmentCodecs, false); + consumer.flush(flushState); + newSegment.setHasVectors(flushState.hasVectors); if (infoStream != null) { - SegmentInfo si = new SegmentInfo(flushState.segmentName, - flushState.numDocs, - directory, false, - hasProx(), - getCodec(), - hasVectors); - - final long newSegmentSize = si.sizeInBytes(true); - String message = " ramUsed=" + nf.format(((double) bytesUsed.get())/1024./1024.) + " MB" + - " newFlushedSize=" + newSegmentSize + - " docs/MB=" + nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) + - " new/old=" + nf.format(100.0*newSegmentSize/bytesUsed.get()) + "%"; - message(message); + message("new segment has " + (flushState.hasVectors ? "vectors" : "no vectors")); + message("flushedFiles=" + newSegment.files()); + message("flushed codecs=" + newSegment.getSegmentCodecs()); } - flushedDocCount += flushState.numDocs; doAfterFlush(); - // Create new SegmentInfo, but do not add to our - // segmentInfos until deletes are flushed - // successfully. - SegmentInfo newSegment = new SegmentInfo(flushState.segmentName, - flushState.numDocs, - directory, false, - hasProx(), - getCodec(), - hasVectors); - - - IndexWriter.setDiagnostics(newSegment, "flush"); success = true; return newSegment; diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index ef1a3b7476f..77fb798af60 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -42,9 +42,9 @@ public final class FieldInfos { // whenever you add a new format, make it 1 smaller (negative version logic)! static final int FORMAT_CURRENT = FORMAT_PER_FIELD_CODEC; - + static final int FORMAT_MINIMUM = FORMAT_START; - + static final byte IS_INDEXED = 0x1; static final byte STORE_TERMVECTOR = 0x2; static final byte STORE_POSITIONS_WITH_TERMVECTOR = 0x4; @@ -52,7 +52,7 @@ public final class FieldInfos { static final byte OMIT_NORMS = 0x10; static final byte STORE_PAYLOADS = 0x20; static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; - + private final ArrayList byNumber = new ArrayList(); private final HashMap byName = new HashMap(); private int format; @@ -111,16 +111,16 @@ public final class FieldInfos { } return false; } - + /** * Add fields that are indexed. Whether they have termvectors has to be specified. - * + * * @param names The names of the fields * @param storeTermVectors Whether the fields store term vectors or not * @param storePositionWithTermVector true if positions should be stored. * @param storeOffsetWithTermVector true if offsets should be stored */ - synchronized public void addIndexed(Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, + synchronized public void addIndexed(Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector) { for (String name : names) { add(name, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector); @@ -129,10 +129,10 @@ public final class FieldInfos { /** * Assumes the fields are not storing term vectors. - * + * * @param names The names of the fields * @param isIndexed Whether the fields are indexed or not - * + * * @see #add(String, boolean) */ synchronized public void add(Collection names, boolean isIndexed) { @@ -143,7 +143,7 @@ public final class FieldInfos { /** * Calls 5 parameter add with false for all TermVector parameters. - * + * * @param name The name of the Fieldable * @param isIndexed true if the field is indexed * @see #add(String, boolean, boolean, boolean, boolean) @@ -154,7 +154,7 @@ public final class FieldInfos { /** * Calls 5 parameter add with false for term vector positions and offsets. - * + * * @param name The name of the field * @param isIndexed true if the field is indexed * @param storeTermVector true if the term vector should be stored @@ -162,12 +162,12 @@ public final class FieldInfos { synchronized public void add(String name, boolean isIndexed, boolean storeTermVector){ add(name, isIndexed, storeTermVector, false, false, false); } - + /** If the field is not yet known, adds it. If it is known, checks to make * sure that the isIndexed flag is the same as was given previously for this * field. If not - marks it as being indexed. Same goes for the TermVector * parameters. - * + * * @param name The name of the field * @param isIndexed true if the field is indexed * @param storeTermVector true if the term vector should be stored @@ -197,7 +197,7 @@ public final class FieldInfos { add(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, false, false); } - + /** If the field is not yet known, adds it. If it is known, checks to make * sure that the isIndexed flag is the same as was given previously for this * field. If not - marks it as being indexed. Same goes for the TermVector @@ -231,8 +231,15 @@ public final class FieldInfos { fi.omitTermFreqAndPositions); } + synchronized public void update(FieldInfos otherInfos) { + int numFields = otherInfos.size(); + for (int i = 0; i < numFields; i++) { + add(otherInfos.fieldInfo(i)); + } + } + private FieldInfo addInternal(String name, boolean isIndexed, - boolean storeTermVector, boolean storePositionWithTermVector, + boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) { name = StringHelper.intern(name); FieldInfo fi = new FieldInfo(name, isIndexed, byNumber.size(), storeTermVector, storePositionWithTermVector, @@ -253,11 +260,11 @@ public final class FieldInfos { /** * Return the fieldName identified by its number. - * + * * @param fieldNumber * @return the fieldName or an empty string when the field * with the given number doesn't exist. - */ + */ public String fieldName(int fieldNumber) { FieldInfo fi = fieldInfo(fieldNumber); return (fi != null) ? fi.name : ""; @@ -268,7 +275,7 @@ public final class FieldInfos { * @param fieldNumber * @return the FieldInfo object or null when the given fieldNumber * doesn't exist. - */ + */ public FieldInfo fieldInfo(int fieldNumber) { return (fieldNumber >= 0) ? byNumber.get(fieldNumber) : null; } @@ -353,7 +360,7 @@ public final class FieldInfos { if (input.getFilePointer() != input.length()) { throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length()); - } + } } } diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index b4b5fc86c1a..c2a7158c9e9 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -222,7 +222,7 @@ public class IndexWriter implements Closeable { final SegmentInfos segmentInfos; // the segments private DocumentsWriter docWriter; - final IndexFileDeleter deleter; + private final IndexFileDeleter deleter; private Set segmentsToOptimize = new HashSet(); // used by optimize to note those needing optimization private int optimizeMaxNumSegments; @@ -1875,10 +1875,10 @@ public class IndexWriter implements Closeable { mergePolicy.close(); mergeScheduler.close(); - bufferedDeletes.clear(); - synchronized(this) { + bufferedDeletes.clear(); + if (pendingCommit != null) { pendingCommit.rollbackCommit(directory); deleter.decRef(pendingCommit); @@ -2047,9 +2047,51 @@ public class IndexWriter implements Closeable { deleter.checkpoint(segmentInfos, false); } - synchronized void addNewSegment(SegmentInfo newSegment) throws IOException { - segmentInfos.add(newSegment); - checkpoint(); + void addFlushedSegment(SegmentInfo newSegment) throws IOException { + assert newSegment != null; + + setDiagnostics(newSegment, "flush"); + + if (useCompoundFile(newSegment)) { + String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); + message("creating compound file " + compoundFileName); + // Now build compound file + boolean success = false; + try { + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); + for(String fileName : newSegment.files()) { + cfsWriter.addFile(fileName); + } + + // Perform the merge + cfsWriter.close(); + synchronized(this) { + deleter.deleteNewFiles(newSegment.files()); + } + + newSegment.setUseCompoundFile(true); + + success = true; + } finally { + if (!success) { + if (infoStream != null) { + message("hit exception " + + "reating compound file for newly flushed segment " + newSegment.name); + } + + synchronized(this) { + deleter.refresh(newSegment.name); + } + } + } + + + } + + synchronized(this) { + segmentInfos.add(newSegment); + checkpoint(); + } } synchronized boolean useCompoundFile(SegmentInfo segmentInfo) throws IOException { @@ -2207,24 +2249,33 @@ public class IndexWriter implements Closeable { } } - /** Merges the provided indexes into this index. - *

After this completes, the index is optimized.

- *

The provided IndexReaders are not closed.

+ /** + * Merges the provided indexes into this index. + *

+ * After this completes, the index is optimized. + *

+ *

+ * The provided IndexReaders are not closed. + *

* - *

NOTE: while this is running, any attempts to - * add or delete documents (with another thread) will be - * paused until this method completes. + *

+ * NOTE: while this is running, any attempts to add or delete documents + * (with another thread) will be paused until this method completes. * - *

See {@link #addIndexes} for details on transactional - * semantics, temporary free space required in the Directory, - * and non-CFS segments on an Exception.

+ *

+ * See {@link #addIndexes} for details on transactional semantics, temporary + * free space required in the Directory, and non-CFS segments on an Exception. + *

* - *

NOTE: if this method hits an OutOfMemoryError - * you should immediately close the writer. See above for details.

+ *

+ * NOTE: if this method hits an OutOfMemoryError you should immediately + * close the writer. See above for details. + *

* - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error + * @throws CorruptIndexException + * if the index is corrupt + * @throws IOException + * if there is a low-level IO error */ public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException { ensureOpen(); @@ -3239,12 +3290,12 @@ public class IndexWriter implements Closeable { // For test purposes. final int getBufferedDeleteTermsSize() { - return docWriter.getPendingDeletes().terms.size(); + return docWriter.getBufferedDeleteTermsSize(); } // For test purposes. final int getNumBufferedDeleteTerms() { - return docWriter.getPendingDeletes().numTermDeletes.get(); + return docWriter.getNumBufferedDeleteTerms(); } // utility routines for tests diff --git a/lucene/src/java/org/apache/lucene/index/SegmentDeletes.java b/lucene/src/java/org/apache/lucene/index/SegmentDeletes.java index 1bb7f028c44..ee54d06a3a4 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentDeletes.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentDeletes.java @@ -102,7 +102,7 @@ class SegmentDeletes { return s; } } - + void update(SegmentDeletes in, boolean noLimit) { numTermDeletes.addAndGet(in.numTermDeletes.get()); for (Map.Entry ent : in.terms.entrySet()) { @@ -168,7 +168,7 @@ class SegmentDeletes { bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.bytes.length); } } - + void clear() { terms.clear(); queries.clear(); @@ -176,12 +176,12 @@ class SegmentDeletes { numTermDeletes.set(0); bytesUsed.set(0); } - + void clearDocIDs() { bytesUsed.addAndGet(-docIDs.size()*BYTES_PER_DEL_DOCID); docIDs.clear(); } - + boolean any() { return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0; } From 3f4c9213c2d4ed1a16a50bfed5d4ad6dc256067c Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Mon, 17 Jan 2011 09:50:51 +0000 Subject: [PATCH 017/200] LUCENE-2324: cleanup segment files when aborting in segment flush) git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1059837 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/java/org/apache/lucene/index/DocumentsWriter.java | 5 ++--- .../org/apache/lucene/index/DocumentsWriterPerThread.java | 6 ++++++ lucene/src/java/org/apache/lucene/index/IndexWriter.java | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 3e69a313d7e..4746952fb9c 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -338,7 +338,6 @@ final class DocumentsWriter { success = true; } finally { - notifyAll(); if (infoStream != null) { message("docWriter: done abort; abortedFiles=" + abortedFiles + " success=" + success); } @@ -395,9 +394,8 @@ final class DocumentsWriter { return false; } - synchronized void close() { + void close() { closed = true; - notifyAll(); } boolean updateDocument(final Document doc, final Analyzer analyzer, final Term delTerm) @@ -512,6 +510,7 @@ final class DocumentsWriter { ThreadState perThread = threadsIterator.next(); perThread.lock(); try { + DocumentsWriterPerThread dwpt = perThread.perThread; final int numDocs = dwpt.getNumDocsInRAM(); diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 0689a3a0e71..d0144d1e2a7 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -305,6 +305,12 @@ public class DocumentsWriterPerThread { return newSegment; } finally { if (!success) { + if (segment != null) { + synchronized(parent.indexWriter) { + parent.indexWriter.deleter.refresh(segment); + } + } + abort(); } } diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index c2a7158c9e9..95ef0b98f52 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -222,7 +222,7 @@ public class IndexWriter implements Closeable { final SegmentInfos segmentInfos; // the segments private DocumentsWriter docWriter; - private final IndexFileDeleter deleter; + final IndexFileDeleter deleter; private Set segmentsToOptimize = new HashSet(); // used by optimize to note those needing optimization private int optimizeMaxNumSegments; From a7875a8228b2de4023fe0bbcc2ecd3df322297e0 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Mon, 24 Jan 2011 19:16:18 +0000 Subject: [PATCH 018/200] add missing files to RT branch git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1062931 13f79535-47bb-0310-9956-ffa450edef68 --- dev-tools/maven/lucene/contrib/ant/pom.xml | 115 +++ .../maven/lucene/contrib/db/bdb-je/pom.xml | 76 ++ dev-tools/maven/lucene/contrib/db/bdb/pom.xml | 87 ++ dev-tools/maven/lucene/contrib/db/pom.xml | 49 ++ dev-tools/maven/lucene/contrib/demo/pom.xml | 124 +++ .../maven/lucene/contrib/highlighter/pom.xml | 84 ++ .../maven/lucene/contrib/instantiated/pom.xml | 72 ++ dev-tools/maven/lucene/contrib/lucli/pom.xml | 108 +++ dev-tools/maven/lucene/contrib/memory/pom.xml | 74 ++ dev-tools/maven/lucene/contrib/misc/pom.xml | 121 +++ dev-tools/maven/lucene/contrib/pom.xml | 62 ++ .../maven/lucene/contrib/queries/pom.xml | 78 ++ .../maven/lucene/contrib/queryparser/pom.xml | 79 ++ .../maven/lucene/contrib/spatial/pom.xml | 77 ++ .../maven/lucene/contrib/spellchecker/pom.xml | 77 ++ dev-tools/maven/lucene/contrib/swing/pom.xml | 101 +++ .../maven/lucene/contrib/wordnet/pom.xml | 105 +++ .../lucene/contrib/xml-query-parser/pom.xml | 82 ++ dev-tools/maven/lucene/pom.xml | 41 + dev-tools/maven/lucene/src/pom.xml | 126 +++ .../maven/modules/analysis/common/pom.xml | 116 +++ dev-tools/maven/modules/analysis/icu/pom.xml | 96 +++ .../maven/modules/analysis/phonetic/pom.xml | 90 +++ dev-tools/maven/modules/analysis/pom.xml | 52 ++ .../maven/modules/analysis/smartcn/pom.xml | 82 ++ .../maven/modules/analysis/stempel/pom.xml | 82 ++ dev-tools/maven/modules/benchmark/pom.xml | 157 ++++ dev-tools/maven/modules/pom.xml | 49 ++ dev-tools/maven/pom.xml | 690 ++++++++++++++++ .../solr/contrib/analysis-extras/pom.xml | 116 +++ .../maven/solr/contrib/clustering/pom.xml | 103 +++ .../solr/contrib/dataimporthandler/pom.xml | 50 ++ .../dataimporthandler/src/extras/pom.xml | 132 ++++ .../contrib/dataimporthandler/src/pom.xml | 129 +++ .../maven/solr/contrib/extraction/pom.xml | 114 +++ dev-tools/maven/solr/contrib/pom.xml | 51 ++ dev-tools/maven/solr/pom.xml | 94 +++ dev-tools/maven/solr/src/pom.xml | 252 ++++++ dev-tools/maven/solr/src/solrj/pom.xml | 135 ++++ dev-tools/maven/solr/src/webapp/pom.xml | 191 +++++ .../search/highlight/TokenSourcesTest.java | 191 +++++ .../lucene/index/codecs/BlockTermState.java | 55 ++ .../lucene/index/codecs/BlockTermsReader.java | 741 ++++++++++++++++++ .../lucene/index/codecs/BlockTermsWriter.java | 316 ++++++++ .../lucene/util/AttributeReflector.java | 34 + .../TestSimpleAttributeImpl.java | 46 ++ solr/contrib/analysis-extras/CHANGES.txt | 30 + .../org/apache/solr/search/SolrSortField.java | 31 + .../TestPHPSerializedResponseWriter.java | 107 +++ 49 files changed, 6070 insertions(+) create mode 100644 dev-tools/maven/lucene/contrib/ant/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/db/bdb-je/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/db/bdb/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/db/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/demo/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/highlighter/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/instantiated/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/lucli/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/memory/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/misc/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/queries/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/queryparser/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/spatial/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/spellchecker/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/swing/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/wordnet/pom.xml create mode 100644 dev-tools/maven/lucene/contrib/xml-query-parser/pom.xml create mode 100644 dev-tools/maven/lucene/pom.xml create mode 100644 dev-tools/maven/lucene/src/pom.xml create mode 100644 dev-tools/maven/modules/analysis/common/pom.xml create mode 100644 dev-tools/maven/modules/analysis/icu/pom.xml create mode 100644 dev-tools/maven/modules/analysis/phonetic/pom.xml create mode 100644 dev-tools/maven/modules/analysis/pom.xml create mode 100644 dev-tools/maven/modules/analysis/smartcn/pom.xml create mode 100644 dev-tools/maven/modules/analysis/stempel/pom.xml create mode 100755 dev-tools/maven/modules/benchmark/pom.xml create mode 100644 dev-tools/maven/modules/pom.xml create mode 100644 dev-tools/maven/pom.xml create mode 100644 dev-tools/maven/solr/contrib/analysis-extras/pom.xml create mode 100644 dev-tools/maven/solr/contrib/clustering/pom.xml create mode 100644 dev-tools/maven/solr/contrib/dataimporthandler/pom.xml create mode 100644 dev-tools/maven/solr/contrib/dataimporthandler/src/extras/pom.xml create mode 100644 dev-tools/maven/solr/contrib/dataimporthandler/src/pom.xml create mode 100644 dev-tools/maven/solr/contrib/extraction/pom.xml create mode 100644 dev-tools/maven/solr/contrib/pom.xml create mode 100644 dev-tools/maven/solr/pom.xml create mode 100644 dev-tools/maven/solr/src/pom.xml create mode 100644 dev-tools/maven/solr/src/solrj/pom.xml create mode 100644 dev-tools/maven/solr/src/webapp/pom.xml create mode 100644 lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java create mode 100644 lucene/src/java/org/apache/lucene/index/codecs/BlockTermState.java create mode 100644 lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java create mode 100644 lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java create mode 100644 lucene/src/java/org/apache/lucene/util/AttributeReflector.java create mode 100644 lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.java create mode 100644 solr/contrib/analysis-extras/CHANGES.txt create mode 100644 solr/src/java/org/apache/solr/search/SolrSortField.java create mode 100644 solr/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java diff --git a/dev-tools/maven/lucene/contrib/ant/pom.xml b/dev-tools/maven/lucene/contrib/ant/pom.xml new file mode 100644 index 00000000000..3410b4f773f --- /dev/null +++ b/dev-tools/maven/lucene/contrib/ant/pom.xml @@ -0,0 +1,115 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-ant + jar + Lucene Contrib Ant + Ant task to create Lucene indexes + + lucene/contrib/ant + ../../build/contrib/ant + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + jtidy + jtidy + + + org.apache.ant + ant + + + org.apache.ant + ant-junit + test + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + src/resources + + + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.lucene.ant.HtmlDocument + HtmlDocument + + + + + + + diff --git a/dev-tools/maven/lucene/contrib/db/bdb-je/pom.xml b/dev-tools/maven/lucene/contrib/db/bdb-je/pom.xml new file mode 100644 index 00000000000..336ee7c2dc4 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/db/bdb-je/pom.xml @@ -0,0 +1,76 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../../pom.xml + + org.apache.lucene + lucene-bdb-je + jar + Lucene Contrib bdb-je + Berkeley DB based Directory implementation + + lucene/contrib/db/bdb-je + ../../../build/contrib/db/bdb-je + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + com.sleepycat + berkeleydb-je + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/db/bdb/pom.xml b/dev-tools/maven/lucene/contrib/db/bdb/pom.xml new file mode 100644 index 00000000000..4ff4ec2c799 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/db/bdb/pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../../pom.xml + + org.apache.lucene + lucene-bdb + jar + Lucene Contrib bdb + Berkeley DB based Directory implementation + + lucene/contrib/db/bdb + ../../../build/contrib/db/bdb + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + com.sleepycat + berkeleydb + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + true + + + + + diff --git a/dev-tools/maven/lucene/contrib/db/pom.xml b/dev-tools/maven/lucene/contrib/db/pom.xml new file mode 100644 index 00000000000..a267ff9139e --- /dev/null +++ b/dev-tools/maven/lucene/contrib/db/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-db-aggregator + Lucene Database aggregator POM + pom + + bdb + bdb-je + + + ../../build/contrib/db/lucene-db-aggregator + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/dev-tools/maven/lucene/contrib/demo/pom.xml b/dev-tools/maven/lucene/contrib/demo/pom.xml new file mode 100644 index 00000000000..e2e61c46090 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/demo/pom.xml @@ -0,0 +1,124 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-demo + war + Lucene Demos + This is the demo for Apache Lucene Java + + lucene/contrib/demo + ../../build/contrib/demo + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + javax.servlet + servlet-api + provided + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.apache.maven.plugins + maven-war-plugin + + src/jsp + true + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + ${build-directory} + + windows + unix + + + + org.apache.lucene.demo.DeleteFiles + DeleteFiles + + + org.apache.lucene.demo.IndexFiles + IndexFiles + + + org.apache.lucene.demo.IndexHTML + IndexHTML + + + org.apache.lucene.demo.SearchFiles + SearchFiles + + + + + + + diff --git a/dev-tools/maven/lucene/contrib/highlighter/pom.xml b/dev-tools/maven/lucene/contrib/highlighter/pom.xml new file mode 100644 index 00000000000..7df7e4affca --- /dev/null +++ b/dev-tools/maven/lucene/contrib/highlighter/pom.xml @@ -0,0 +1,84 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-highlighter + jar + Lucene Highlighter + + This is the highlighter for apache lucene java + + + lucene/contrib/highlighter + ../../build/contrib/highlighter + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-memory + ${project.version} + + + ${project.groupId} + lucene-queries + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/instantiated/pom.xml b/dev-tools/maven/lucene/contrib/instantiated/pom.xml new file mode 100644 index 00000000000..f40c5544d16 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/instantiated/pom.xml @@ -0,0 +1,72 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-instantiated + jar + Lucene InstantiatedIndex + InstantiatedIndex, alternative RAM store for small corpora. + + lucene/contrib/instantiated + ../../build/contrib/instantiated + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/lucli/pom.xml b/dev-tools/maven/lucene/contrib/lucli/pom.xml new file mode 100644 index 00000000000..f6266f7caa2 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/lucli/pom.xml @@ -0,0 +1,108 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-lucli + jar + Lucene Lucli + Lucene Command Line Interface + + lucene/contrib/lucli + ../../build/contrib/lucli + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + jline + jline + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + true + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + lucli.Lucli + lucli + + + + + + + diff --git a/dev-tools/maven/lucene/contrib/memory/pom.xml b/dev-tools/maven/lucene/contrib/memory/pom.xml new file mode 100644 index 00000000000..940fdf1b34e --- /dev/null +++ b/dev-tools/maven/lucene/contrib/memory/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-memory + jar + Lucene Memory + + High-performance single-document index to compare against Query + + + lucene/contrib/memory + ../../build/contrib/memory + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/misc/pom.xml b/dev-tools/maven/lucene/contrib/misc/pom.xml new file mode 100644 index 00000000000..62bf4577e53 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/misc/pom.xml @@ -0,0 +1,121 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-misc + jar + Lucene Miscellaneous + Miscellaneous Lucene extensions + + lucene/contrib/misc + ../../build/contrib/misc + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.lucene.index.FieldNormModifier + FieldNormModifier + + + org.apache.lucene.index.IndexSplitter + IndexSplitter + + + org.apache.lucene.index.MultiPassIndexSplitter + MultiPassIndexSplitter + + + org.apache.lucene.misc.GetTermInfo + GetTermInfo + + + org.apache.lucene.misc.HighFreqTerms + HighFreqTerms + + + org.apache.lucene.misc.IndexMergeTool + IndexMergeTool + + + org.apache.lucene.misc.LengthNormModifier + LengthNormModifier + + + + + + + diff --git a/dev-tools/maven/lucene/contrib/pom.xml b/dev-tools/maven/lucene/contrib/pom.xml new file mode 100644 index 00000000000..039b87d80a7 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../pom.xml + + org.apache.lucene + lucene-contrib-aggregator + Lucene Contrib aggregator POM + pom + + ant + db + demo + highlighter + instantiated + lucli + memory + misc + queries + queryparser + spatial + spellchecker + swing + wordnet + xml-query-parser + + + build/lucene-contrib-aggregator + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/dev-tools/maven/lucene/contrib/queries/pom.xml b/dev-tools/maven/lucene/contrib/queries/pom.xml new file mode 100644 index 00000000000..4cef0b9fa3b --- /dev/null +++ b/dev-tools/maven/lucene/contrib/queries/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-queries + jar + Lucene Queries + + Queries - various query object exotica not in core + + + lucene/contrib/queries + ../../build/contrib/queries + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + jakarta-regexp + jakarta-regexp + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/queryparser/pom.xml b/dev-tools/maven/lucene/contrib/queryparser/pom.xml new file mode 100644 index 00000000000..6f215fa962d --- /dev/null +++ b/dev-tools/maven/lucene/contrib/queryparser/pom.xml @@ -0,0 +1,79 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-queryparser + jar + Lucene Query Parser + + This is the Flexible Query Parser for apache lucene java + + + lucene/contrib/queryparser + ../../build/contrib/queryparser + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + src/resources + + + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/spatial/pom.xml b/dev-tools/maven/lucene/contrib/spatial/pom.xml new file mode 100644 index 00000000000..6bac5c10a8f --- /dev/null +++ b/dev-tools/maven/lucene/contrib/spatial/pom.xml @@ -0,0 +1,77 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-spatial + jar + Lucene Spatial + Spatial search package + + lucene/contrib/spatial + ../../build/contrib/spatial + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-queries + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/spellchecker/pom.xml b/dev-tools/maven/lucene/contrib/spellchecker/pom.xml new file mode 100644 index 00000000000..5be1474a734 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/spellchecker/pom.xml @@ -0,0 +1,77 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-spellchecker + jar + Lucene Spellchecker + Spell Checker + + lucene/contrib/spellchecker + ../../build/contrib/spellchecker + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/contrib/swing/pom.xml b/dev-tools/maven/lucene/contrib/swing/pom.xml new file mode 100644 index 00000000000..1f2f4e5e424 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/swing/pom.xml @@ -0,0 +1,101 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-swing + jar + Lucene Swing + Swing Models + + lucene/contrib/swing + ../../build/contrib/swing + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.lucene.swing.models.ListSearcherSimulator + ListSearchSimulator + + + org.apache.lucene.swing.models.TableSearcherSimulator + TableSearchSimulator + + + + + + + diff --git a/dev-tools/maven/lucene/contrib/wordnet/pom.xml b/dev-tools/maven/lucene/contrib/wordnet/pom.xml new file mode 100644 index 00000000000..d55557322b6 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/wordnet/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-wordnet + jar + Lucene Wordnet + WordNet + + lucene/contrib/wordnet + ../../build/contrib/wordnet + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.lucene.wordnet.SynExpand + SynExpand + + + org.apache.lucene.wordnet.SynExpand + SynExpand + + + org.apache.lucene.wordnet.Syns2Index + Syns2Index + + + + + + + diff --git a/dev-tools/maven/lucene/contrib/xml-query-parser/pom.xml b/dev-tools/maven/lucene/contrib/xml-query-parser/pom.xml new file mode 100644 index 00000000000..f4c9aea1dd7 --- /dev/null +++ b/dev-tools/maven/lucene/contrib/xml-query-parser/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.lucene + lucene-xml-query-parser + jar + Lucene XML Query Parser + XML query parser + + lucene/contrib/xml-query-parser + ../../build/contrib/xml-query-parser + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-queries + ${project.version} + + + javax.servlet + servlet-api + provided + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/lucene/pom.xml b/dev-tools/maven/lucene/pom.xml new file mode 100644 index 00000000000..42b7c546157 --- /dev/null +++ b/dev-tools/maven/lucene/pom.xml @@ -0,0 +1,41 @@ + + + 4.0.0 + + org.apache.lucene + lucene-solr-grandparent + 4.0-SNAPSHOT + ../pom.xml + + org.apache.lucene + lucene-parent + pom + Lucene parent POM + Lucene parent POM + + src + contrib + + + build/lucene-parent + + diff --git a/dev-tools/maven/lucene/src/pom.xml b/dev-tools/maven/lucene/src/pom.xml new file mode 100644 index 00000000000..43c83e76750 --- /dev/null +++ b/dev-tools/maven/lucene/src/pom.xml @@ -0,0 +1,126 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../pom.xml + + org.apache.lucene + lucene-core + jar + Lucene Core + Apache Lucene Java Core + + lucene + ../build + + + + junit + junit + test + + + org.apache.ant + ant + test + + + org.apache.ant + ant-junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + java + test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${project.version} + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.lucene.index.CheckIndex + CheckIndex + + + org.apache.lucene.index.IndexReader + IndexReader + + + org.apache.lucene.store.LockStressTest + LockStressTest + + + org.apache.lucene.store.LockVerifyServer + IndexReader + + + org.apache.lucene.util.English + English + + + + + + + diff --git a/dev-tools/maven/modules/analysis/common/pom.xml b/dev-tools/maven/modules/analysis/common/pom.xml new file mode 100644 index 00000000000..8977178b874 --- /dev/null +++ b/dev-tools/maven/modules/analysis/common/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../../lucene/pom.xml + + org.apache.lucene + lucene-analyzers-common + jar + Lucene Common Analyzers + Additional Analyzers + + modules/analysis/common + ../build/common + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + src/resources + + + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.lucene.analysis.charfilter.HtmlStripCharFilter + HtmlStripCharFilter + + + org.apache.lucene.analysis.en.PorterStemmer + EnglishPorterStemmer + + + org.tartarus.snowball.TestApp + SnowballTestApp + + + + + + + diff --git a/dev-tools/maven/modules/analysis/icu/pom.xml b/dev-tools/maven/modules/analysis/icu/pom.xml new file mode 100644 index 00000000000..1ce2183627e --- /dev/null +++ b/dev-tools/maven/modules/analysis/icu/pom.xml @@ -0,0 +1,96 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../../lucene/pom.xml + + org.apache.lucene + lucene-analyzers-icu + jar + Lucene ICU Analysis Components + + Provides integration with ICU (International Components for Unicode) for + stronger Unicode and internationalization support. + + + modules/analysis/icu + ../build/icu + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + test-jar + test + + + com.ibm.icu + icu4j + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + src/resources + + + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/modules/analysis/phonetic/pom.xml b/dev-tools/maven/modules/analysis/phonetic/pom.xml new file mode 100644 index 00000000000..665752e5300 --- /dev/null +++ b/dev-tools/maven/modules/analysis/phonetic/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../../lucene/pom.xml + + org.apache.lucene + lucene-analyzers-phonetic + jar + Lucene Phonetic Filters + + Provides phonetic encoding via Commons Codec. + + + modules/analysis/phonetic + ../build/phonetic + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + test-jar + test + + + commons-codec + commons-codec + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/modules/analysis/pom.xml b/dev-tools/maven/modules/analysis/pom.xml new file mode 100644 index 00000000000..7829ed61b77 --- /dev/null +++ b/dev-tools/maven/modules/analysis/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../lucene/pom.xml + + org.apache.lucene + lucene-analysis-modules-aggregator + Lucene Analysis Modules aggregator POM + pom + + common + icu + phonetic + smartcn + stempel + + + build/lucene-analysis-modules-aggregator + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/dev-tools/maven/modules/analysis/smartcn/pom.xml b/dev-tools/maven/modules/analysis/smartcn/pom.xml new file mode 100644 index 00000000000..10a1a013dea --- /dev/null +++ b/dev-tools/maven/modules/analysis/smartcn/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../../lucene/pom.xml + + org.apache.lucene + lucene-analyzers-smartcn + jar + Lucene Smart Chinese Analyzer + Smart Chinese Analyzer + + modules/analysis/smartcn + ../build/smartcn + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + src/resources + + + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/modules/analysis/stempel/pom.xml b/dev-tools/maven/modules/analysis/stempel/pom.xml new file mode 100644 index 00000000000..19b6907e127 --- /dev/null +++ b/dev-tools/maven/modules/analysis/stempel/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../../lucene/pom.xml + + org.apache.lucene + lucene-analyzers-stempel + jar + Lucene Stempel Analyzer + Stempel Analyzer + + modules/analysis/stempel + ../build/stempel + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + src/resources + + + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + diff --git a/dev-tools/maven/modules/benchmark/pom.xml b/dev-tools/maven/modules/benchmark/pom.xml new file mode 100755 index 00000000000..e4b35e338c4 --- /dev/null +++ b/dev-tools/maven/modules/benchmark/pom.xml @@ -0,0 +1,157 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../../lucene/pom.xml + + org.apache.lucene + lucene-benchmark + jar + Lucene Benchmark + Lucene Benchmarking Module + + modules/benchmark + build + + + + ${project.groupId} + lucene-core + ${project.version} + + + ${project.groupId} + lucene-core + ${project.version} + test-jar + test + + + ${project.groupId} + lucene-analyzers-common + ${project.version} + + + ${project.groupId} + lucene-demo + ${project.version} + classes + + + ${project.groupId} + lucene-highlighter + ${project.version} + + + ${project.groupId} + lucene-memory + ${project.version} + + + commons-beanutils + commons-beanutils + + + commons-collections + commons-collections + + + org.apache.commons + commons-compress + + + commons-digester + commons-digester + + + commons-logging + commons-logging + + + xerces + xercesImpl + + + xml-apis + xml-apis + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes/java + ${build-directory}/classes/test + src/java + src/test + + + ${project.build.testSourceDirectory} + + **/*.java + + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.lucene.benchmark.byTask.Benchmark + Benchmark + + + org.apache.lucene.benchmark.quality.trec.QueryDriver + QueryDriver + + + org.apache.lucene.benchmark.quality.utils.QualityQueriesFinder + QualityQueriesFinder + + + org.apache.lucene.benchmark.utils.ExtractReuters + ExtractReuters + + + org.apache.lucene.benchmark.utils.ExtractWikipedia + ExtractWikipedia + + + + + + + diff --git a/dev-tools/maven/modules/pom.xml b/dev-tools/maven/modules/pom.xml new file mode 100644 index 00000000000..1b60080c172 --- /dev/null +++ b/dev-tools/maven/modules/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + org.apache.lucene + lucene-parent + 4.0-SNAPSHOT + ../lucene/pom.xml + + org.apache.lucene + lucene-modules-aggregator + Lucene Modules aggregator POM + pom + + analysis + benchmark + + + build/lucene-modules-aggregator + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/dev-tools/maven/pom.xml b/dev-tools/maven/pom.xml new file mode 100644 index 00000000000..c0c5ae7d812 --- /dev/null +++ b/dev-tools/maven/pom.xml @@ -0,0 +1,690 @@ + + + 4.0.0 + + org.apache + apache + 8 + + org.apache.lucene + lucene-solr-grandparent + 4.0-SNAPSHOT + pom + Grandparent POM for Apache Lucene Java and Apache Solr + Parent POM for Apache Lucene Java and Apache Solr + http://lucene.apache.org/java + + lucene + modules + solr + + + 4.0.0 + yyyy-MM-dd HH:mm:ss + 1.5 + + + JIRA + http://issues.apache.org/jira/browse/LUCENE + + + Hudson + http://lucene.zones.apache.org:8080/hudson/job/Lucene-Nightly/ + + + + General List + general-subscribe@lucene.apache.org + general-unsubscribe@lucene.apache.org + + http://mail-archives.apache.org/mod_mbox/lucene-general/ + + + + Java User List + java-user-subscribe@lucene.apache.org + java-user-unsubscribe@lucene.apache.org + + http://mail-archives.apache.org/mod_mbox/lucene-java-user/ + + + + Java Developer List + dev-subscribe@lucene.apache.org + dev-unsubscribe@lucene.apache.org + http://mail-archives.apache.org/mod_mbox/lucene-dev/ + + + Java Commits List + commits-subscribe@lucene.apache.org + commits-unsubscribe@lucene.apache.org + + http://mail-archives.apache.org/mod_mbox/lucene-java-commits/ + + + + 2000 + + + scm:svn:http://svn.apache.org/repos/asf/lucene/dev/trunk/${module-directory} + + + scm:svn:https://svn.apache.org/repos/asf/lucene/dev/trunk/${module-directory} + + + http://svn.apache.org/viewvc/lucene/dev/trunk/${module-directory} + + + + + Apache 2 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + carrot2.org + Carrot2 Maven2 repository + http://download.carrot2.org/maven2/ + + never + + + + apache.snapshots + Apache Snapshot Repository + http://repository.apache.org/snapshots + + false + + + never + + + + + + + com.ibm.icu + icu4j + 4.6 + + + com.sleepycat + berkeleydb + 4.7.25 + + + com.sleepycat + berkeleydb-je + 3.3.93 + + + commons-beanutils + commons-beanutils + 1.7.0 + + + commons-codec + commons-codec + 1.4 + + + commons-collections + commons-collections + 3.2.1 + + + commons-digester + commons-digester + 1.7 + + + commons-fileupload + commons-fileupload + 1.2.1 + + + commons-io + commons-io + 1.4 + + + commons-httpclient + commons-httpclient + 3.1 + + + commons-lang + commons-lang + 2.4 + + + commons-logging + commons-logging + 1.1.1 + + + jakarta-regexp + jakarta-regexp + 1.4 + + + javax.activation + activation + 1.1 + + + javax.mail + mail + 1.4.1 + + + jline + jline + 0.9.1 + + + jtidy + jtidy + 4aug2000r7-dev + + + junit + junit + 4.7 + + + org.apache.ant + ant + 1.7.1 + + + org.apache.ant + ant-junit + 1.7.1 + + + org.apache.commons + commons-compress + 1.1 + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + 1.0.1 + + + org.apache.solr + solr-commons-csv + ${project.version} + + + org.apache.solr + solr-noggit + ${project.version} + + + org.apache.tika + tika-core + 0.8 + + + org.apache.tika + tika-parsers + 0.8 + + + org.apache.velocity + velocity + 1.6.4 + + + org.apache.velocity + velocity-tools + 2.0 + + + org.apache.zookeeper + zookeeper + 3.3.1 + + + org.carrot2 + carrot2-core + 3.4.2 + + + org.codehaus.woodstox + wstx-asl + 3.2.7 + + + org.easymock + easymock + 2.2 + + + org.mortbay.jetty + jetty + 6.1.26 + + + org.mortbay.jetty + jetty-util + 6.1.26 + + + org.mortbay.jetty + jsp-2.1-glassfish + 2.1.v20091210 + + + org.mortbay.jetty + jsp-2.1-jetty + 6.1.26 + + + org.mortbay.jetty + jsp-api-2.1-glassfish + 2.1.v20091210 + + + org.slf4j + jcl-over-slf4j + 1.5.5 + + + org.slf4j + log4j-over-slf4j + 1.5.5 + + + org.slf4j + slf4j-api + 1.5.5 + + + org.slf4j + slf4j-jdk14 + 1.5.5 + + + xerces + xercesImpl + 2.10.0 + + + xml-apis + xml-apis + 2.10.0 + + + javax.servlet + servlet-api + 2.4 + + + + + lucene/build/lucene-parent + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.6 + + + org.apache.maven.plugins + maven-clean-plugin + 2.4.1 + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + ${java.compat.version} + ${java.compat.version} + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.5 + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.0 + + + org.apache.maven.plugins + maven-install-plugin + 2.3.1 + + + org.apache.maven.plugins + maven-jar-plugin + 2.3.1 + + + + + + + + ${project.groupId} + ${project.groupId} + ${project.name} + + ${base.specification.version}.${now.version} + The Apache Software Foundation + + ${project.version} ${svn.revision} - ${user.name} - ${now.timestamp} + The Apache Software Foundation + ${java.compat.version} + ${java.compat.version} + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.4.3 + + + org.apache.maven.plugins + maven-surefire-plugin + 2.7.1 + + plain + ${project.build.testOutputDirectory} + + temp + 1 + ${tests.codec} + ${tests.directory} + ${tests.iter} + ${tests.locale} + ${tests.luceneMatchVersion} + ${tests.multiplier} + ${tests.nightly} + ${tests.seed} + ${tests.timezone} + + + + + org.apache.maven.plugins + maven-war-plugin + 2.1.1 + + + + + + + + ${project.groupId} + ${project.groupId} + ${project.name} + + ${base.specification.version}.${now.version} + The Apache Software Foundation + + ${project.version} ${svn.revision} - ${user.name} - ${now.timestamp} + The Apache Software Foundation + ${java.compat.version} + ${java.compat.version} + + + + + + org.codehaus.mojo + appassembler-maven-plugin + 1.1 + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + org.codehaus.mojo + buildnumber-maven-plugin + 1.0-beta-4 + + + org.mortbay.jetty + maven-jetty-plugin + 6.1.26 + + + org.codehaus.gmaven + gmaven-plugin + 1.3 + + + + + + org.codehaus.gmaven + gmaven-plugin + + + generate-timestamps + validate + + execute + + + + project.properties['now.timestamp'] = "${maven.build.timestamp}" + project.properties['now.version'] = ("${maven.build.timestamp}" =~ /[- :]/).replaceAll(".") + project.properties['now.year'] = "${maven.build.timestamp}".substring(0, 4) + + + + + + + org.codehaus.mojo + buildnumber-maven-plugin + + + validate + + create + + + + + false + false + true + svn.revision + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-java-compat-version-and-maven-2.2.1 + + enforce + + + + + Java ${java.compat.version}+ is required. + [${java.compat.version},) + + + Maven 2.2.1+ is required. + [2.2.1,) + + + + + + + + + + + + bootstrap + + + + org.apache.maven.plugins + maven-antrun-plugin + + + get-jars-and-poms + install + + run + + + + + + + + + + + + org.apache.maven.plugins + maven-install-plugin + + + install-icu4j + install + + install-file + + + com.ibm.icu + icu4j + 4.6 + jar + modules/analysis/icu/lib/icu4j-4_6.jar + + + + install-xercesImpl + install + + install-file + + + xerces + xercesImpl + 2.10.0 + jar + modules/benchmark/lib/xercesImpl-2.10.0.jar + + + + install-xml-apis + install + + install-file + + + xml-apis + xml-apis + 2.10.0 + jar + modules/benchmark/lib/xml-apis-2.10.0.jar + + + + install-berkeleydb + install + + install-file + + + com.sleepycat + berkeleydb + 4.7.25 + jar + lucene/contrib/db/bdb/lib/db-4.7.25.jar + + + + install-berkeleydb-je + install + + install-file + + + com.sleepycat + berkeleydb-je + 3.3.93 + jar + lucene/contrib/db/bdb-je/lib/je-3.3.93.jar + + + + install-solr-commons-csv + install + + install-file + + + org.apache.solr + solr-commons-csv + ${project.version} + jar + solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar + + + + install-solr-noggit + install + + install-file + + + org.apache.solr + solr-noggit + ${project.version} + jar + solr/lib/apache-solr-noggit-r944541.jar + + + + + + + + + diff --git a/dev-tools/maven/solr/contrib/analysis-extras/pom.xml b/dev-tools/maven/solr/contrib/analysis-extras/pom.xml new file mode 100644 index 00000000000..ce67bbaf22e --- /dev/null +++ b/dev-tools/maven/solr/contrib/analysis-extras/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.solr + solr-analysis-extras + jar + Apache Solr Analysis Extras + Apache Solr Analysis Extras + + solr/contrib/analysis-extras + build + 4.0 + + + + ${project.groupId} + solr-core + ${project.version} + + + ${project.groupId} + solr-core + ${project.version} + test-jar + test + + + org.apache.lucene + lucene-analyzers-common + ${project.version} + + + org.apache.lucene + lucene-analyzers-icu + ${project.version} + + + org.apache.lucene + lucene-analyzers-smartcn + ${project.version} + + + org.apache.lucene + lucene-analyzers-stempel + ${project.version} + + + org.apache.lucene + lucene-core + ${project.version} + + + org.apache.lucene + lucene-core + ${project.version} + test-jar + test + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes + ${build-directory}/test-classes + src/java + src/test + + + test-files + + + ../../src/test-files + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ../../../../testlogging.properties + + + + + + diff --git a/dev-tools/maven/solr/contrib/clustering/pom.xml b/dev-tools/maven/solr/contrib/clustering/pom.xml new file mode 100644 index 00000000000..81266fd9cda --- /dev/null +++ b/dev-tools/maven/solr/contrib/clustering/pom.xml @@ -0,0 +1,103 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.solr + solr-clustering + jar + Apache Solr Clustering + Apache Solr Clustering + + solr/contrib/clustering + build + 4.0 + + + + ${project.groupId} + solr-core + ${project.version} + + + ${project.groupId} + solr-core + ${project.version} + test-jar + test + + + ${project.groupId} + solr-solrj + ${project.version} + + + org.apache.lucene + lucene-analyzers-common + ${project.version} + + + org.apache.lucene + lucene-core + ${project.version} + test-jar + test + + + org.carrot2 + carrot2-core + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes + ${build-directory}/test-classes + + + src/test/resources + + + ../../src/test-files + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ../../../../testlogging.properties + + + + + + diff --git a/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml b/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml new file mode 100644 index 00000000000..a90b1dec722 --- /dev/null +++ b/dev-tools/maven/solr/contrib/dataimporthandler/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.solr + solr-dataimporthandler-aggregator + pom + Apache Solr DataImportHandler aggregator POM + Apache Solr DataImportHandler aggregator POM + + src + src/extras + + + target/solr-dataimporthandler-aggregator + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/dev-tools/maven/solr/contrib/dataimporthandler/src/extras/pom.xml b/dev-tools/maven/solr/contrib/dataimporthandler/src/extras/pom.xml new file mode 100644 index 00000000000..bae3817983f --- /dev/null +++ b/dev-tools/maven/solr/contrib/dataimporthandler/src/extras/pom.xml @@ -0,0 +1,132 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../../../pom.xml + + org.apache.solr + solr-dataimporthandler-extras + jar + Apache Solr DataImportHandler Extras + Apache Solr DataImportHandler Extras + + solr/contrib/dataimporthandler/src/extras + ../../target/extras + 4.0 + + + + ${project.groupId} + solr-core + ${project.version} + + + ${project.groupId} + solr-core + ${project.version} + test-jar + test + + + ${project.groupId} + solr-dataimporthandler + ${project.version} + + + ${project.groupId} + solr-dataimporthandler + ${project.version} + test-jar + test + + + ${project.groupId} + solr-solrj + ${project.version} + + + org.apache.lucene + lucene-core + ${project.version} + test-jar + test + + + commons-io + commons-io + + + javax.activation + activation + + + javax.mail + mail + + + org.apache.tika + tika-parsers + + + org.slf4j + slf4j-api + + + xerces + xercesImpl + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/extras/classes + ${build-directory}/extras/test-classes + main/java + test/java + + + test/resources + + + ../../../../src/test-files + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ../../../../../../testlogging.properties + + + + + + diff --git a/dev-tools/maven/solr/contrib/dataimporthandler/src/pom.xml b/dev-tools/maven/solr/contrib/dataimporthandler/src/pom.xml new file mode 100644 index 00000000000..0745d2e5417 --- /dev/null +++ b/dev-tools/maven/solr/contrib/dataimporthandler/src/pom.xml @@ -0,0 +1,129 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../../pom.xml + + org.apache.solr + solr-dataimporthandler + jar + Apache Solr DataImportHandler + Apache Solr DataImportHandler + + solr/contrib/dataimporthandler + ../target + 4.0 + + + + ${project.groupId} + solr-core + ${project.version} + + + ${project.groupId} + solr-core + ${project.version} + test-jar + test + + + ${project.groupId} + solr-solrj + ${project.version} + + + org.apache.lucene + lucene-analyzers-common + ${project.version} + + + org.apache.lucene + lucene-core + ${project.version} + test-jar + test + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + + commons-io + commons-io + + + org.slf4j + slf4j-api + + + junit + junit + test + + + org.easymock + easymock + test + + + + ${build-directory} + ${build-directory}/classes + ${build-directory}/test-classes + main/java + test/java + + + test/resources + + + ../../../src/test-files + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ../../../../../testlogging.properties + + + + + + diff --git a/dev-tools/maven/solr/contrib/extraction/pom.xml b/dev-tools/maven/solr/contrib/extraction/pom.xml new file mode 100644 index 00000000000..75d1f2316c9 --- /dev/null +++ b/dev-tools/maven/solr/contrib/extraction/pom.xml @@ -0,0 +1,114 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.solr + solr-cell + jar + Apache Solr Content Extraction Library + + Apache Solr Content Extraction Library integrates Apache Tika + content extraction framework into Solr + + + solr/contrib/extraction + build + 4.0 + + + + ${project.groupId} + solr-core + ${project.version} + + + ${project.groupId} + solr-core + ${project.version} + test-jar + test + + + ${project.groupId} + solr-solrj + ${project.version} + + + org.apache.lucene + lucene-analyzers-common + ${project.version} + + + org.apache.lucene + lucene-core + ${project.version} + test-jar + test + + + com.ibm.icu + icu4j + + + org.apache.tika + tika-parsers + + + xerces + xercesImpl + + + junit + junit + test + + + + ${build-directory} + ${build-directory}/classes + ${build-directory}/test-classes + + + src/test/resources + + + ../../src/test-files + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ../../../../testlogging.properties + + + + + + diff --git a/dev-tools/maven/solr/contrib/pom.xml b/dev-tools/maven/solr/contrib/pom.xml new file mode 100644 index 00000000000..fff3350f474 --- /dev/null +++ b/dev-tools/maven/solr/contrib/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../pom.xml + + org.apache.solr + solr-contrib-aggregator + Apache Solr Contrib aggregator POM + pom + + analysis-extras + clustering + dataimporthandler + extraction + + + ../build/solr-contrib-aggregator + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/dev-tools/maven/solr/pom.xml b/dev-tools/maven/solr/pom.xml new file mode 100644 index 00000000000..465af72a1a0 --- /dev/null +++ b/dev-tools/maven/solr/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + org.apache.lucene + lucene-solr-grandparent + 4.0-SNAPSHOT + ../pom.xml + + org.apache.solr + solr-parent + pom + Apache Solr parent POM + Apache Solr parent POM + + src + src/solrj + src/webapp + contrib + + + 1.6 + + + JIRA + http://issues.apache.org/jira/browse/SOLR + + + Hudson + + http://lucene.zones.apache.org:8080/hudson/job/Solr-Nightly/ + + + + + Solr User List + solr-user-subscribe@lucene.apache.org + solr-user-unsubscribe@lucene.apache.org + + http://mail-archives.apache.org/mod_mbox/solr-user/ + + + + Java Developer List + dev-subscribe@lucene.apache.org + dev-unsubscribe@lucene.apache.org + http://mail-archives.apache.org/mod_mbox/lucene-dev/ + + + Java Commits List + commits-subscribe@lucene.apache.org + commits-unsubscribe@lucene.apache.org + + http://mail-archives.apache.org/mod_mbox/lucene-java-commits/ + + + + 2006 + + build/solr-parent + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + ${project.name} ${project.version} API (${now.version}) + ${project.name} ${project.version} API (${now.version}) + + + + + + diff --git a/dev-tools/maven/solr/src/pom.xml b/dev-tools/maven/solr/src/pom.xml new file mode 100644 index 00000000000..0014b38bfd1 --- /dev/null +++ b/dev-tools/maven/solr/src/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../pom.xml + + org.apache.solr + solr-core + jar + Apache Solr Core + Apache Solr Core + + solr + ../build + 4.0 + + + + ${project.groupId} + solr-solrj + ${project.version} + + + ${project.groupId} + solr-noggit + + + org.apache.lucene + lucene-core + ${project.version} + test-jar + test + + + org.apache.lucene + lucene-analyzers-common + ${project.version} + + + org.apache.lucene + lucene-analyzers-phonetic + ${project.version} + + + org.apache.lucene + lucene-highlighter + ${project.version} + + + org.apache.lucene + lucene-memory + ${project.version} + + + org.apache.lucene + lucene-misc + ${project.version} + + + org.apache.lucene + lucene-queries + ${project.version} + + + org.apache.lucene + lucene-spatial + ${project.version} + + + org.apache.lucene + lucene-spellchecker + ${project.version} + + + org.apache.solr + solr-commons-csv + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + + commons-codec + commons-codec + + + commons-fileupload + commons-fileupload + + + commons-httpclient + commons-httpclient + + + commons-io + commons-io + + + commons-lang + commons-lang + + + org.apache.velocity + velocity + + + org.apache.velocity + velocity-tools + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + org.mortbay.jetty + jsp-2.1-jetty + provided + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-jdk14 + + + javax.servlet + servlet-api + provided + + + junit + junit + test + + + org.easymock + easymock + test + + + + ${build-directory} + ${build-directory}/solr + ${build-directory}/tests + java + test + + + test-files + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + webapp/src + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ../../../testlogging.properties + + + + + org.codehaus.mojo + appassembler-maven-plugin + + -Xmx128M + flat + + windows + unix + + + + org.apache.solr.client.solrj.embedded.JettySolrRunner + JettySolrRunner + + + org.apache.solr.util.BitSetPerf + BitSetPerf + -Xms128m -Xbatch + + + org.apache.solr.util.SimplePostTool + SimplePostTool + + + org.apache.solr.util.SuggestMissingFactories + SuggestMissingFactories + + + + + + + diff --git a/dev-tools/maven/solr/src/solrj/pom.xml b/dev-tools/maven/solr/src/solrj/pom.xml new file mode 100644 index 00000000000..262793a6dbe --- /dev/null +++ b/dev-tools/maven/solr/src/solrj/pom.xml @@ -0,0 +1,135 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.solr + solr-solrj + jar + Apache Solr Solrj + Apache Solr Solrj + + solr/src/solrj + ../../build/solrj + 4.0 + + + + org.apache.lucene + lucene-core + ${project.version} + + + org.apache.lucene + lucene-core + ${project.version} + test-jar + test + + + org.apache.lucene + lucene-analyzers-common + ${project.version} + test + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + + org.apache.zookeeper + zookeeper + + + javax.jms + jms + + + com.sun.jmx + jmxri + + + com.sun.jdmk + jmxtools + + + + + commons-httpclient + commons-httpclient + + + commons-io + commons-io + + + org.slf4j + slf4j-api + + + junit + junit + test + + + + ${build-directory} + ${build-directory} + . + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + ../common + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + true + + + + + diff --git a/dev-tools/maven/solr/src/webapp/pom.xml b/dev-tools/maven/solr/src/webapp/pom.xml new file mode 100644 index 00000000000..af914c5a94a --- /dev/null +++ b/dev-tools/maven/solr/src/webapp/pom.xml @@ -0,0 +1,191 @@ + + + 4.0.0 + + org.apache.solr + solr-parent + 4.0-SNAPSHOT + ../../pom.xml + + org.apache.solr + solr + war + Apache Solr Search Server + Apache Solr Search Server + + solr/src/webapp + ../../build/web + + + + ${project.groupId} + solr-core + ${project.version} + + + commons-chain + commons-chain + + + commons-digester + commons-digester + + + commons-logging + commons-logging + + + commons-validator + commons-validator + + + jakarta-regexp + jakarta-regexp + + + dom4j + dom4j + + + log4j + log4j + + + javax.activation + activation + + + javax.mail + mail + + + jline + jline + + + org.apache.struts + struts-core + + + org.apache.struts + struts-taglib + + + org.apache.struts + struts-tiles + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + oro + oro + + + sslext + sslext + + + xml-apis + xml-apis + + + + + ${project.groupId} + solr-dataimporthandler + ${project.version} + + + org.codehaus.woodstox + wstx-asl + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + log4j-over-slf4j + + + + ${build-directory} + + + org.apache.maven.plugins + maven-javadoc-plugin + + true + + + + org.apache.maven.plugins + maven-source-plugin + + + + + + false + + + + org.apache.maven.plugins + maven-war-plugin + + web + web/WEB-INF/web.xml + + + ../../contrib/dataimporthandler/src/main/webapp + + + + + + org.mortbay.jetty + maven-jetty-plugin + + 10 + + + 8080 + 60000 + + + + / + + web,../../contrib/dataimporthandler/src/main/webapp + + + + + + + diff --git a/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java b/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java new file mode 100644 index 00000000000..3fcaa10b535 --- /dev/null +++ b/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java @@ -0,0 +1,191 @@ +package org.apache.lucene.search.highlight; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermPositionVector; +import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.LuceneTestCase; + +// LUCENE-2874 +public class TokenSourcesTest extends LuceneTestCase { + private static final String FIELD = "text"; + + private static final class OverlapAnalyzer extends Analyzer { + + @Override + public TokenStream tokenStream(String fieldName, Reader reader) { + return new TokenStreamOverlap(); + } + } + + private static final class TokenStreamOverlap extends TokenStream { + private Token[] tokens; + + private int i = -1; + + private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class); + private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); + + public TokenStreamOverlap() { + reset(); + } + + @Override + public boolean incrementToken() throws IOException { + this.i++; + if (this.i >= this.tokens.length) { + return false; + } + clearAttributes(); + termAttribute.setEmpty().append(this.tokens[i]); + offsetAttribute.setOffset(this.tokens[i].startOffset(), + this.tokens[i].endOffset()); + positionIncrementAttribute.setPositionIncrement(this.tokens[i] + .getPositionIncrement()); + return true; + } + + @Override + public void reset() { + this.i = -1; + this.tokens = new Token[] { + new Token(new char[] { 't', 'h', 'e' }, 0, 3, 0, 3), + new Token(new char[] { '{', 'f', 'o', 'x', '}' }, 0, 5, 0, 7), + new Token(new char[] { 'f', 'o', 'x' }, 0, 3, 4, 7), + new Token(new char[] { 'd', 'i', 'd' }, 0, 3, 8, 11), + new Token(new char[] { 'n', 'o', 't' }, 0, 3, 12, 15), + new Token(new char[] { 'j', 'u', 'm', 'p' }, 0, 4, 16, 20) }; + this.tokens[1].setPositionIncrement(0); + } + } + + public void testOverlapWithOffset() throws CorruptIndexException, + LockObtainFailedException, IOException, InvalidTokenOffsetsException { + final String TEXT = "the fox did not jump"; + final Directory directory = newDirectory(); + final IndexWriter indexWriter = new IndexWriter(directory, + newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer())); + try { + final Document document = new Document(); + document.add(new Field(FIELD, new TokenStreamOverlap(), + TermVector.WITH_OFFSETS)); + indexWriter.addDocument(document); + } finally { + indexWriter.close(); + } + final IndexReader indexReader = IndexReader.open(directory, true); + try { + assertEquals(1, indexReader.numDocs()); + final IndexSearcher indexSearcher = new IndexSearcher(indexReader); + try { + final DisjunctionMaxQuery query = new DisjunctionMaxQuery(1); + query.add(new SpanTermQuery(new Term(FIELD, "{fox}"))); + query.add(new SpanTermQuery(new Term(FIELD, "fox"))); + // final Query phraseQuery = new SpanNearQuery(new SpanQuery[] { + // new SpanTermQuery(new Term(FIELD, "{fox}")), + // new SpanTermQuery(new Term(FIELD, "fox")) }, 0, true); + + TopDocs hits = indexSearcher.search(query, 1); + assertEquals(1, hits.totalHits); + final Highlighter highlighter = new Highlighter( + new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), + new QueryScorer(query)); + final TokenStream tokenStream = TokenSources + .getTokenStream( + (TermPositionVector) indexReader.getTermFreqVector(0, FIELD), + false); + assertEquals("the fox did not jump", + highlighter.getBestFragment(tokenStream, TEXT)); + } finally { + indexSearcher.close(); + } + } finally { + indexReader.close(); + directory.close(); + } + } + + public void testOverlapWithPositionsAndOffset() throws CorruptIndexException, + LockObtainFailedException, IOException, InvalidTokenOffsetsException { + final String TEXT = "the fox did not jump"; + final Directory directory = newDirectory(); + final IndexWriter indexWriter = new IndexWriter(directory, + newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer())); + try { + final Document document = new Document(); + document.add(new Field(FIELD, new TokenStreamOverlap(), + TermVector.WITH_POSITIONS_OFFSETS)); + indexWriter.addDocument(document); + } finally { + indexWriter.close(); + } + final IndexReader indexReader = IndexReader.open(directory, true); + try { + assertEquals(1, indexReader.numDocs()); + final IndexSearcher indexSearcher = new IndexSearcher(indexReader); + try { + final DisjunctionMaxQuery query = new DisjunctionMaxQuery(1); + query.add(new SpanTermQuery(new Term(FIELD, "{fox}"))); + query.add(new SpanTermQuery(new Term(FIELD, "fox"))); + // final Query phraseQuery = new SpanNearQuery(new SpanQuery[] { + // new SpanTermQuery(new Term(FIELD, "{fox}")), + // new SpanTermQuery(new Term(FIELD, "fox")) }, 0, true); + + TopDocs hits = indexSearcher.search(query, 1); + assertEquals(1, hits.totalHits); + final Highlighter highlighter = new Highlighter( + new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), + new QueryScorer(query)); + final TokenStream tokenStream = TokenSources + .getTokenStream( + (TermPositionVector) indexReader.getTermFreqVector(0, FIELD), + false); + assertEquals("the fox did not jump", + highlighter.getBestFragment(tokenStream, TEXT)); + } finally { + indexSearcher.close(); + } + } finally { + indexReader.close(); + directory.close(); + } + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermState.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermState.java new file mode 100644 index 00000000000..4ab22aef72f --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermState.java @@ -0,0 +1,55 @@ +package org.apache.lucene.index.codecs; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.OrdTermState; +import org.apache.lucene.index.TermState; + +/** + * Holds all state required for {@link PostingsReaderBase} + * to produce a {@link DocsEnum} without re-seeking the + * terms dict. + */ +public class BlockTermState extends OrdTermState { + public int docFreq; // how many docs have this term + public long totalTermFreq; // total number of occurrences of this term + + public int termCount; // term ord are in the current block + public long blockFilePointer; // fp into the terms dict primary file (_X.tib) that holds this term + + public int blockTermCount; // how many terms in current block + + @Override + public void copyFrom(TermState _other) { + assert _other instanceof BlockTermState : "can not copy from " + _other.getClass().getName(); + BlockTermState other = (BlockTermState) _other; + super.copyFrom(_other); + docFreq = other.docFreq; + totalTermFreq = other.totalTermFreq; + termCount = other.termCount; + blockFilePointer = other.blockFilePointer; + + // NOTE: don't copy blockTermCount; + // it's "transient": used only by the "primary" + // termState, and regenerated on seek by TermState + } + + @Override + public String toString() { + return super.toString() + "ord=" + ord + " docFreq=" + docFreq + " totalTermFreq=" + totalTermFreq + " termCount=" + termCount + " blockFP=" + blockFilePointer; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java new file mode 100644 index 00000000000..1ea93a04495 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java @@ -0,0 +1,741 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.TreeMap; + +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.DoubleBarrelLRUCache; + +/** Handles a terms dict, but decouples all details of + * doc/freqs/positions reading to an instance of {@link + * PostingsReaderBase}. This class is reusable for + * codecs that use a different format for + * docs/freqs/positions (though codecs are also free to + * make their own terms dict impl). + * + *

This class also interacts with an instance of {@link + * TermsIndexReaderBase}, to abstract away the specific + * implementation of the terms dict index. + * @lucene.experimental */ + +public class BlockTermsReader extends FieldsProducer { + // Open input to the main terms dict file (_X.tis) + private final IndexInput in; + + // Reads the terms dict entries, to gather state to + // produce DocsEnum on demand + private final PostingsReaderBase postingsReader; + + private final TreeMap fields = new TreeMap(); + + // Comparator that orders our terms + private final Comparator termComp; + + // Caches the most recently looked-up field + terms: + private final DoubleBarrelLRUCache termsCache; + + // Reads the terms index + private TermsIndexReaderBase indexReader; + + // keeps the dirStart offset + protected long dirOffset; + + // Used as key for the terms cache + private static class FieldAndTerm extends DoubleBarrelLRUCache.CloneableKey { + String field; + BytesRef term; + + public FieldAndTerm() { + } + + public FieldAndTerm(FieldAndTerm other) { + field = other.field; + term = new BytesRef(other.term); + } + + @Override + public boolean equals(Object _other) { + FieldAndTerm other = (FieldAndTerm) _other; + return other.field == field && term.bytesEquals(other.term); + } + + @Override + public Object clone() { + return new FieldAndTerm(this); + } + + @Override + public int hashCode() { + return field.hashCode() * 31 + term.hashCode(); + } + } + + private String segment; + + public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize, + Comparator termComp, int termsCacheSize, String codecId) + throws IOException { + + this.postingsReader = postingsReader; + termsCache = new DoubleBarrelLRUCache(termsCacheSize); + + this.termComp = termComp; + this.segment = segment; + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTermsWriter.TERMS_EXTENSION), + readBufferSize); + + boolean success = false; + try { + readHeader(in); + + // Have PostingsReader init itself + postingsReader.init(in); + + // Read per-field details + seekDir(in, dirOffset); + + final int numFields = in.readVInt(); + + for(int i=0;i= 0; + final long termsStartPointer = in.readVLong(); + final FieldInfo fieldInfo = fieldInfos.fieldInfo(field); + final long sumTotalTermFreq = fieldInfo.omitTermFreqAndPositions ? -1 : in.readVLong(); + assert !fields.containsKey(fieldInfo.name); + fields.put(fieldInfo.name, new FieldReader(fieldInfo, numTerms, termsStartPointer, sumTotalTermFreq)); + } + success = true; + } finally { + if (!success) { + in.close(); + } + } + + this.indexReader = indexReader; + } + + protected void readHeader(IndexInput input) throws IOException { + CodecUtil.checkHeader(in, BlockTermsWriter.CODEC_NAME, + BlockTermsWriter.VERSION_START, + BlockTermsWriter.VERSION_CURRENT); + dirOffset = in.readLong(); + } + + protected void seekDir(IndexInput input, long dirOffset) + throws IOException { + input.seek(dirOffset); + } + + @Override + public void loadTermsIndex(int indexDivisor) throws IOException { + indexReader.loadTermsIndex(indexDivisor); + } + + @Override + public void close() throws IOException { + try { + try { + if (indexReader != null) { + indexReader.close(); + } + } finally { + // null so if an app hangs on to us (ie, we are not + // GCable, despite being closed) we still free most + // ram + indexReader = null; + if (in != null) { + in.close(); + } + } + } finally { + try { + if (postingsReader != null) { + postingsReader.close(); + } + } finally { + for(FieldReader field : fields.values()) { + field.close(); + } + } + } + } + + public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, BlockTermsWriter.TERMS_EXTENSION)); + } + + public static void getExtensions(Collection extensions) { + extensions.add(BlockTermsWriter.TERMS_EXTENSION); + } + + @Override + public FieldsEnum iterator() { + return new TermFieldsEnum(); + } + + @Override + public Terms terms(String field) throws IOException { + return fields.get(field); + } + + // Iterates through all fields + private class TermFieldsEnum extends FieldsEnum { + final Iterator it; + FieldReader current; + + TermFieldsEnum() { + it = fields.values().iterator(); + } + + @Override + public String next() { + if (it.hasNext()) { + current = it.next(); + return current.fieldInfo.name; + } else { + current = null; + return null; + } + } + + @Override + public TermsEnum terms() throws IOException { + return current.iterator(); + } + } + + private class FieldReader extends Terms implements Closeable { + final long numTerms; + final FieldInfo fieldInfo; + final long termsStartPointer; + final long sumTotalTermFreq; + + FieldReader(FieldInfo fieldInfo, long numTerms, long termsStartPointer, long sumTotalTermFreq) { + assert numTerms > 0; + this.fieldInfo = fieldInfo; + this.numTerms = numTerms; + this.termsStartPointer = termsStartPointer; + this.sumTotalTermFreq = sumTotalTermFreq; + } + + @Override + public Comparator getComparator() { + return termComp; + } + + @Override + public void close() { + super.close(); + } + + @Override + public TermsEnum iterator() throws IOException { + return new SegmentTermsEnum(); + } + + @Override + public long getUniqueTermCount() { + return numTerms; + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + // Iterates through terms in this field + private final class SegmentTermsEnum extends TermsEnum { + private final IndexInput in; + private final BlockTermState state; + private final boolean doOrd; + private final FieldAndTerm fieldTerm = new FieldAndTerm(); + private final TermsIndexReaderBase.FieldIndexEnum indexEnum; + private final BytesRef term = new BytesRef(); + + /* This is true if indexEnum is "still" seek'd to the index term + for the current term. We set it to true on seeking, and then it + remains valid until next() is called enough times to load another + terms block: */ + private boolean indexIsCurrent; + + /* True if we've already called .next() on the indexEnum, to "bracket" + the current block of terms: */ + private boolean didIndexNext; + + /* Next index term, bracketing the current block of terms; this is + only valid if didIndexNext is true: */ + private BytesRef nextIndexTerm; + + /* True after seek(TermState), do defer seeking. If the app then + calls next() (which is not "typical"), then we'll do the real seek */ + private boolean seekPending; + + /* How many blocks we've read since last seek. Once this + is >= indexEnum.getDivisor() we set indexIsCurrent to false (since + the index can no long bracket seek-within-block). */ + private int blocksSinceSeek; + + private byte[] termSuffixes; + private ByteArrayDataInput termSuffixesReader = new ByteArrayDataInput(null); + + /* Common prefix used for all terms in this block. */ + private int termBlockPrefix; + + private byte[] docFreqBytes; + private final ByteArrayDataInput freqReader = new ByteArrayDataInput(null); + private int metaDataUpto; + + public SegmentTermsEnum() throws IOException { + in = (IndexInput) BlockTermsReader.this.in.clone(); + in.seek(termsStartPointer); + indexEnum = indexReader.getFieldEnum(fieldInfo); + doOrd = indexReader.supportsOrd(); + fieldTerm.field = fieldInfo.name; + state = postingsReader.newTermState(); + state.totalTermFreq = -1; + state.ord = -1; + + termSuffixes = new byte[128]; + docFreqBytes = new byte[64]; + //System.out.println("BTR.enum init this=" + this + " postingsReader=" + postingsReader); + } + + @Override + public Comparator getComparator() { + return termComp; + } + + @Override + public SeekStatus seek(final BytesRef target, final boolean useCache) throws IOException { + + if (indexEnum == null) { + throw new IllegalStateException("terms index was not loaded"); + } + + //System.out.println("BTR.seek seg=" + segment + " target=" + fieldInfo.name + ":" + target.utf8ToString() + " " + target + " current=" + term().utf8ToString() + " " + term() + " useCache=" + useCache + " indexIsCurrent=" + indexIsCurrent + " didIndexNext=" + didIndexNext + " seekPending=" + seekPending + " divisor=" + indexReader.getDivisor() + " this=" + this); + /* + if (didIndexNext) { + if (nextIndexTerm == null) { + //System.out.println(" nextIndexTerm=null"); + } else { + //System.out.println(" nextIndexTerm=" + nextIndexTerm.utf8ToString()); + } + } + */ + + // Check cache + if (useCache) { + fieldTerm.term = target; + // TODO: should we differentiate "frozen" + // TermState (ie one that was cloned and + // cached/returned by termState()) from the + // malleable (primary) one? + final TermState cachedState = termsCache.get(fieldTerm); + if (cachedState != null) { + seekPending = true; + //System.out.println(" cached!"); + seek(target, cachedState); + //System.out.println(" term=" + term.utf8ToString()); + return SeekStatus.FOUND; + } + } + + boolean doSeek = true; + + // See if we can avoid seeking, because target term + // is after current term but before next index term: + if (indexIsCurrent) { + + final int cmp = termComp.compare(term, target); + + if (cmp == 0) { + // Already at the requested term + return SeekStatus.FOUND; + } else if (cmp < 0) { + + // Target term is after current term + if (!didIndexNext) { + if (indexEnum.next() == -1) { + nextIndexTerm = null; + } else { + nextIndexTerm = indexEnum.term(); + } + //System.out.println(" now do index next() nextIndexTerm=" + (nextIndexTerm == null ? "null" : nextIndexTerm.utf8ToString())); + didIndexNext = true; + } + + if (nextIndexTerm == null || termComp.compare(target, nextIndexTerm) < 0) { + // Optimization: requested term is within the + // same term block we are now in; skip seeking + // (but do scanning): + doSeek = false; + //System.out.println(" skip seek: nextIndexTerm=" + (nextIndexTerm == null ? "null" : nextIndexTerm.utf8ToString())); + } + } + } + + if (doSeek) { + //System.out.println(" seek"); + + // Ask terms index to find biggest indexed term (= + // first term in a block) that's <= our text: + in.seek(indexEnum.seek(target)); + boolean result = nextBlock(); + + // Block must exist since, at least, the indexed term + // is in the block: + assert result; + + indexIsCurrent = true; + didIndexNext = false; + blocksSinceSeek = 0; + + if (doOrd) { + state.ord = indexEnum.ord()-1; + } + + // NOTE: the first _next() after an index seek is + // a bit wasteful, since it redundantly reads some + // suffix bytes into the buffer. We could avoid storing + // those bytes in the primary file, but then when + // next()ing over an index term we'd have to + // special case it: + term.copy(indexEnum.term()); + //System.out.println(" seek: term=" + term.utf8ToString()); + } else { + ////System.out.println(" skip seek"); + } + + seekPending = false; + + // Now scan: + while (_next() != null) { + final int cmp = termComp.compare(term, target); + if (cmp == 0) { + // Match! + if (useCache) { + // Store in cache + decodeMetaData(); + termsCache.put(new FieldAndTerm(fieldTerm), (BlockTermState) state.clone()); + } + //System.out.println(" FOUND"); + return SeekStatus.FOUND; + } else if (cmp > 0) { + //System.out.println(" NOT_FOUND term=" + term.utf8ToString()); + return SeekStatus.NOT_FOUND; + } + + // The purpose of the terms dict index is to seek + // the enum to the closest index term before the + // term we are looking for. So, we should never + // cross another index term (besides the first + // one) while we are scanning: + assert indexIsCurrent; + } + + indexIsCurrent = false; + //System.out.println(" END"); + return SeekStatus.END; + } + + @Override + public BytesRef next() throws IOException { + //System.out.println("BTR.next() seekPending=" + seekPending + " pendingSeekCount=" + state.termCount); + + // If seek was previously called and the term was cached, + // usually caller is just going to pull a D/&PEnum or get + // docFreq, etc. But, if they then call next(), + // this method catches up all internal state so next() + // works properly: + if (seekPending) { + assert !indexIsCurrent; + in.seek(state.blockFilePointer); + final int pendingSeekCount = state.termCount; + boolean result = nextBlock(); + + final long savOrd = state.ord; + + // Block must exist since seek(TermState) was called w/ a + // TermState previously returned by this enum when positioned + // on a real term: + assert result; + + while(state.termCount < pendingSeekCount) { + BytesRef nextResult = _next(); + assert nextResult != null; + } + seekPending = false; + state.ord = savOrd; + } + return _next(); + } + + /* Decodes only the term bytes of the next term. If caller then asks for + metadata, ie docFreq, totalTermFreq or pulls a D/&PEnum, we then (lazily) + decode all metadata up to the current term. */ + private BytesRef _next() throws IOException { + //System.out.println("BTR._next this=" + this + " termCount=" + state.termCount + " (vs " + state.blockTermCount + ")"); + if (state.termCount == state.blockTermCount) { + if (!nextBlock()) { + //System.out.println(" eof"); + indexIsCurrent = false; + return null; + } + } + + // TODO: cutover to something better for these ints! simple64? + final int suffix = termSuffixesReader.readVInt(); + //System.out.println(" suffix=" + suffix); + + term.length = termBlockPrefix + suffix; + if (term.bytes.length < term.length) { + term.grow(term.length); + } + termSuffixesReader.readBytes(term.bytes, termBlockPrefix, suffix); + state.termCount++; + + // NOTE: meaningless in the non-ord case + state.ord++; + + //System.out.println(" return term=" + fieldInfo.name + ":" + term.utf8ToString() + " " + term); + return term; + } + + @Override + public BytesRef term() { + return term; + } + + @Override + public int docFreq() throws IOException { + //System.out.println("BTR.docFreq"); + decodeMetaData(); + //System.out.println(" return " + state.docFreq); + return state.docFreq; + } + + @Override + public long totalTermFreq() throws IOException { + decodeMetaData(); + return state.totalTermFreq; + } + + @Override + public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + //System.out.println("BTR.docs this=" + this); + decodeMetaData(); + //System.out.println(" state.docFreq=" + state.docFreq); + final DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, skipDocs, reuse); + assert docsEnum != null; + return docsEnum; + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + //System.out.println("BTR.d&p this=" + this); + decodeMetaData(); + if (fieldInfo.omitTermFreqAndPositions) { + return null; + } else { + DocsAndPositionsEnum dpe = postingsReader.docsAndPositions(fieldInfo, state, skipDocs, reuse); + //System.out.println(" return d&pe=" + dpe); + return dpe; + } + } + + @Override + public void seek(BytesRef target, TermState otherState) throws IOException { + //System.out.println("BTR.seek termState target=" + target.utf8ToString() + " " + target + " this=" + this); + assert otherState != null && otherState instanceof BlockTermState; + assert !doOrd || ((BlockTermState) otherState).ord < numTerms; + state.copyFrom(otherState); + seekPending = true; + indexIsCurrent = false; + term.copy(target); + } + + @Override + public TermState termState() throws IOException { + //System.out.println("BTR.termState this=" + this); + decodeMetaData(); + TermState ts = (TermState) state.clone(); + //System.out.println(" return ts=" + ts); + return ts; + } + + @Override + public SeekStatus seek(long ord) throws IOException { + //System.out.println("BTR.seek by ord ord=" + ord); + if (indexEnum == null) { + throw new IllegalStateException("terms index was not loaded"); + } + + if (ord >= numTerms) { + state.ord = numTerms-1; + return SeekStatus.END; + } + + // TODO: if ord is in same terms block and + // after current ord, we should avoid this seek just + // like we do in the seek(BytesRef) case + in.seek(indexEnum.seek(ord)); + boolean result = nextBlock(); + + // Block must exist since ord < numTerms: + assert result; + + indexIsCurrent = true; + didIndexNext = false; + blocksSinceSeek = 0; + seekPending = false; + + state.ord = indexEnum.ord()-1; + assert state.ord >= -1: "ord=" + state.ord; + term.copy(indexEnum.term()); + + // Now, scan: + int left = (int) (ord - state.ord); + while(left > 0) { + final BytesRef term = _next(); + assert term != null; + left--; + assert indexIsCurrent; + } + + // always found + return SeekStatus.FOUND; + } + + public long ord() { + if (!doOrd) { + throw new UnsupportedOperationException(); + } + return state.ord; + } + + private void doPendingSeek() { + } + + /* Does initial decode of next block of terms; this + doesn't actually decode the docFreq, totalTermFreq, + postings details (frq/prx offset, etc.) metadata; + it just loads them as byte[] blobs which are then + decoded on-demand if the metadata is ever requested + for any term in this block. This enables terms-only + intensive consumes (eg certain MTQs, respelling) to + not pay the price of decoding metadata they won't + use. */ + private boolean nextBlock() throws IOException { + + // TODO: we still lazy-decode the byte[] for each + // term (the suffix), but, if we decoded + // all N terms up front then seeking could do a fast + // bsearch w/in the block... + + //System.out.println("BTR.nextBlock() fp=" + in.getFilePointer() + " this=" + this); + state.blockFilePointer = in.getFilePointer(); + state.blockTermCount = in.readVInt(); + //System.out.println(" blockTermCount=" + state.blockTermCount); + if (state.blockTermCount == 0) { + return false; + } + termBlockPrefix = in.readVInt(); + + // term suffixes: + int len = in.readVInt(); + if (termSuffixes.length < len) { + termSuffixes = new byte[ArrayUtil.oversize(len, 1)]; + } + //System.out.println(" termSuffixes len=" + len); + in.readBytes(termSuffixes, 0, len); + termSuffixesReader.reset(termSuffixes); + + // docFreq, totalTermFreq + len = in.readVInt(); + if (docFreqBytes.length < len) { + docFreqBytes = new byte[ArrayUtil.oversize(len, 1)]; + } + //System.out.println(" freq bytes len=" + len); + in.readBytes(docFreqBytes, 0, len); + freqReader.reset(docFreqBytes); + metaDataUpto = 0; + + state.termCount = 0; + + postingsReader.readTermsBlock(in, fieldInfo, state); + + blocksSinceSeek++; + indexIsCurrent &= (blocksSinceSeek < indexReader.getDivisor()); + //System.out.println(" indexIsCurrent=" + indexIsCurrent); + + return true; + } + + private void decodeMetaData() throws IOException { + //System.out.println("BTR.decodeMetadata mdUpto=" + metaDataUpto + " vs termCount=" + state.termCount + " state=" + state); + if (!seekPending) { + // lazily catch up on metadata decode: + final int limit = state.termCount; + state.termCount = metaDataUpto; + while (metaDataUpto < limit) { + //System.out.println(" decode"); + // TODO: we could make "tiers" of metadata, ie, + // decode docFreq/totalTF but don't decode postings + // metadata; this way caller could get + // docFreq/totalTF w/o paying decode cost for + // postings + state.docFreq = freqReader.readVInt(); + if (!fieldInfo.omitTermFreqAndPositions) { + state.totalTermFreq = state.docFreq + freqReader.readVLong(); + } + postingsReader.nextTerm(fieldInfo, state); + metaDataUpto++; + state.termCount++; + } + } else { + //System.out.println(" skip! seekPending"); + } + } + } + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java new file mode 100644 index 00000000000..c60b42506ed --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java @@ -0,0 +1,316 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.RamUsageEstimator; + +// TODO: currently we encode all terms between two indexed +// terms as a block; but, we could decouple the two, ie +// allow several blocks in between two indexed terms + +/** + * Writes terms dict, block-encoding (column stride) each + * term's metadata for each set of terms between two + * index terms. + * + * @lucene.experimental + */ + +public class BlockTermsWriter extends FieldsConsumer { + + final static String CODEC_NAME = "BLOCK_TERMS_DICT"; + + // Initial format + public static final int VERSION_START = 0; + + public static final int VERSION_CURRENT = VERSION_START; + + /** Extension of terms file */ + static final String TERMS_EXTENSION = "tib"; + + protected final IndexOutput out; + final PostingsWriterBase postingsWriter; + final FieldInfos fieldInfos; + FieldInfo currentField; + private final TermsIndexWriterBase termsIndexWriter; + private final List fields = new ArrayList(); + private final Comparator termComp; + private final String segment; + + public BlockTermsWriter( + TermsIndexWriterBase termsIndexWriter, + SegmentWriteState state, + PostingsWriterBase postingsWriter, + Comparator termComp) throws IOException + { + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); + this.termsIndexWriter = termsIndexWriter; + this.termComp = termComp; + out = state.directory.createOutput(termsFileName); + fieldInfos = state.fieldInfos; + writeHeader(out); + currentField = null; + this.postingsWriter = postingsWriter; + segment = state.segmentName; + + //System.out.println("BTW.init seg=" + state.segmentName); + + postingsWriter.start(out); // have consumer write its format/header + } + + protected void writeHeader(IndexOutput out) throws IOException { + CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); + + out.writeLong(0); // leave space for end index pointer + } + + @Override + public TermsConsumer addField(FieldInfo field) throws IOException { + //System.out.println("\nBTW.addField seg=" + segment + " field=" + field.name); + assert currentField == null || currentField.name.compareTo(field.name) < 0; + currentField = field; + TermsIndexWriterBase.FieldWriter fieldIndexWriter = termsIndexWriter.addField(field, out.getFilePointer()); + final TermsWriter terms = new TermsWriter(fieldIndexWriter, field, postingsWriter); + fields.add(terms); + return terms; + } + + @Override + public void close() throws IOException { + + try { + + int nonZeroCount = 0; + for(TermsWriter field : fields) { + if (field.numTerms > 0) { + nonZeroCount++; + } + } + + final long dirStart = out.getFilePointer(); + + out.writeVInt(nonZeroCount); + for(TermsWriter field : fields) { + if (field.numTerms > 0) { + out.writeVInt(field.fieldInfo.number); + out.writeVLong(field.numTerms); + out.writeVLong(field.termsStartPointer); + if (!field.fieldInfo.omitTermFreqAndPositions) { + out.writeVLong(field.sumTotalTermFreq); + } + } + } + writeTrailer(dirStart); + } finally { + try { + out.close(); + } finally { + try { + postingsWriter.close(); + } finally { + termsIndexWriter.close(); + } + } + } + } + + protected void writeTrailer(long dirStart) throws IOException { + // TODO Auto-generated method stub + out.seek(CodecUtil.headerLength(CODEC_NAME)); + out.writeLong(dirStart); + } + + private static class TermEntry { + public final BytesRef term = new BytesRef(); + public TermStats stats; + } + + class TermsWriter extends TermsConsumer { + private final FieldInfo fieldInfo; + private final PostingsWriterBase postingsWriter; + private final long termsStartPointer; + private long numTerms; + private final TermsIndexWriterBase.FieldWriter fieldIndexWriter; + long sumTotalTermFreq; + private final BytesRef lastTerm = new BytesRef(); + + private TermEntry[] pendingTerms; + + private int pendingCount; + + TermsWriter( + TermsIndexWriterBase.FieldWriter fieldIndexWriter, + FieldInfo fieldInfo, + PostingsWriterBase postingsWriter) + { + this.fieldInfo = fieldInfo; + this.fieldIndexWriter = fieldIndexWriter; + pendingTerms = new TermEntry[32]; + for(int i=0;i getComparator() { + return termComp; + } + + @Override + public PostingsConsumer startTerm(BytesRef text) throws IOException { + //System.out.println("BTW.startTerm seg=" + segment + " term=" + fieldInfo.name + ":" + text.utf8ToString() + " " + text); + postingsWriter.startTerm(); + return postingsWriter; + } + + private final BytesRef lastPrevTerm = new BytesRef(); + + @Override + public void finishTerm(BytesRef text, TermStats stats) throws IOException { + + assert stats.docFreq > 0; + //System.out.println("BTW.finishTerm seg=" + segment + " term=" + fieldInfo.name + ":" + text.utf8ToString() + " " + text + " df=" + stats.docFreq); + + final boolean isIndexTerm = fieldIndexWriter.checkIndexTerm(text, stats); + + if (isIndexTerm) { + if (pendingCount > 0) { + // Instead of writing each term, live, we gather terms + // in RAM in a pending buffer, and then write the + // entire block in between index terms: + flushBlock(); + } + fieldIndexWriter.add(text, stats, out.getFilePointer()); + } + + if (pendingTerms.length == pendingCount) { + final TermEntry[] newArray = new TermEntry[ArrayUtil.oversize(pendingCount+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; + System.arraycopy(pendingTerms, 0, newArray, 0, pendingCount); + for(int i=pendingCount;i 0) { + flushBlock(); + } + // EOF marker: + out.writeVInt(0); + + this.sumTotalTermFreq = sumTotalTermFreq; + fieldIndexWriter.finish(out.getFilePointer()); + } + + private int sharedPrefix(BytesRef term1, BytesRef term2) { + assert term1.offset == 0; + assert term2.offset == 0; + int pos1 = 0; + int pos1End = pos1 + Math.min(term1.length, term2.length); + int pos2 = 0; + while(pos1 < pos1End) { + if (term1.bytes[pos1] != term2.bytes[pos2]) { + return pos1; + } + pos1++; + pos2++; + } + return pos1; + } + + private final RAMOutputStream bytesWriter = new RAMOutputStream(); + + private void flushBlock() throws IOException { + //System.out.println("BTW.flushBlock pendingCount=" + pendingCount); + + // First pass: compute common prefix for all terms + // in the block, against term before first term in + // this block: + int commonPrefix = sharedPrefix(lastPrevTerm, pendingTerms[0].term); + for(int termCount=1;termCount attClass, String key, Object value); + +} diff --git a/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.java b/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.java new file mode 100644 index 00000000000..b8e9a0df7e2 --- /dev/null +++ b/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.java @@ -0,0 +1,46 @@ +package org.apache.lucene.analysis.tokenattributes; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.LuceneTestCase; + +import java.util.Collections; +import java.util.HashMap; + +public class TestSimpleAttributeImpl extends LuceneTestCase { + + // this checks using reflection API if the defaults are correct + public void testAttributes() { + _TestUtil.assertAttributeReflection(new PositionIncrementAttributeImpl(), + Collections.singletonMap(PositionIncrementAttribute.class.getName()+"#positionIncrement", 1)); + _TestUtil.assertAttributeReflection(new FlagsAttributeImpl(), + Collections.singletonMap(FlagsAttribute.class.getName()+"#flags", 0)); + _TestUtil.assertAttributeReflection(new TypeAttributeImpl(), + Collections.singletonMap(TypeAttribute.class.getName()+"#type", TypeAttribute.DEFAULT_TYPE)); + _TestUtil.assertAttributeReflection(new PayloadAttributeImpl(), + Collections.singletonMap(PayloadAttribute.class.getName()+"#payload", null)); + _TestUtil.assertAttributeReflection(new KeywordAttributeImpl(), + Collections.singletonMap(KeywordAttribute.class.getName()+"#keyword", false)); + _TestUtil.assertAttributeReflection(new OffsetAttributeImpl(), new HashMap() {{ + put(OffsetAttribute.class.getName()+"#startOffset", 0); + put(OffsetAttribute.class.getName()+"#endOffset", 0); + }}); + } + +} diff --git a/solr/contrib/analysis-extras/CHANGES.txt b/solr/contrib/analysis-extras/CHANGES.txt new file mode 100644 index 00000000000..e512decaa99 --- /dev/null +++ b/solr/contrib/analysis-extras/CHANGES.txt @@ -0,0 +1,30 @@ + Apache Solr - Analysis Extras + Release Notes + +Introduction +------------ +The analysis-extras plugin provides additional analyzers that rely +upon large dependencies/dictionaries. + +It includes integration with ICU for multilingual support, and +analyzers for Chinese and Polish. + + +$Id$ +================== Release 4.0-dev ================== + +(No Changes) + +================== Release 3.1-dev ================== + +* SOLR-2210: Add icu-based tokenizer and filters to contrib/analysis-extras (rmuir) + +* SOLR-1336: Add SmartChinese (word segmentation for Simplified Chinese) + tokenizer and filters to contrib/analysis-extras (rmuir) + +* SOLR-2211,LUCENE-2763: Added UAX29URLEmailTokenizerFactory, which implements + UAX#29, a unicode algorithm with good results for most languages, as well as + URL and E-mail tokenization according to the relevant RFCs. + (Tom Burton-West via rmuir) + +* SOLR-2237: Added StempelPolishStemFilterFactory to contrib/analysis-extras (rmuir) diff --git a/solr/src/java/org/apache/solr/search/SolrSortField.java b/solr/src/java/org/apache/solr/search/SolrSortField.java new file mode 100644 index 00000000000..8b21e4357bd --- /dev/null +++ b/solr/src/java/org/apache/solr/search/SolrSortField.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.solr.search; + +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.SortField; + +import java.io.IOException; + +/**@lucene.internal + * + */ +public interface SolrSortField { + public SortField weight(IndexSearcher searcher) throws IOException; +} diff --git a/solr/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java b/solr/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java new file mode 100644 index 00000000000..d67e1fb8cb8 --- /dev/null +++ b/solr/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.response; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.LinkedHashMap; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.response.PHPSerializedResponseWriter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Basic PHPS tests based on JSONWriterTest + * + */ +public class TestPHPSerializedResponseWriter extends SolrTestCaseJ4 { + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml","schema.xml"); + } + + @Test + public void testSimple() throws IOException { + SolrQueryRequest req = req("dummy"); + SolrQueryResponse rsp = new SolrQueryResponse(); + QueryResponseWriter w = new PHPSerializedResponseWriter(); + + StringWriter buf = new StringWriter(); + rsp.add("data1", "hello"); + rsp.add("data2", 42); + rsp.add("data3", true); + w.write(buf, req, rsp); + assertEquals("a:3:{s:5:\"data1\";s:5:\"hello\";s:5:\"data2\";i:42;s:5:\"data3\";b:1;}", + buf.toString()); + req.close(); + } + + + @Test + public void testSolrDocuments() throws IOException { + SolrQueryRequest req = req("q","*:*"); + SolrQueryResponse rsp = new SolrQueryResponse(); + QueryResponseWriter w = new PHPSerializedResponseWriter(); + StringWriter buf = new StringWriter(); + + SolrDocument d = new SolrDocument(); + + SolrDocument d1 = d; + d.addField("id","1"); + d.addField("data1","hello"); + d.addField("data2",42); + d.addField("data3",true); + + // multivalued fields: + + // extremely odd edge case: value is a map + + // we use LinkedHashMap because we are doing a string comparison + // later and we need predictible ordering + LinkedHashMap nl = new LinkedHashMap(); + nl.put("data4.1", "hashmap"); + nl.put("data4.2", "hello"); + d.addField("data4",nl); + // array value + d.addField("data5",Arrays.asList("data5.1", "data5.2", "data5.3")); + + // adding one more document to test array indexes + d = new SolrDocument(); + SolrDocument d2 = d; + d.addField("id","2"); + + SolrDocumentList sdl = new SolrDocumentList(); + sdl.add(d1); + sdl.add(d2); + rsp.add("response", sdl); + + w.write(buf, req, rsp); + assertEquals("a:1:{s:8:\"response\";a:3:{s:8:\"numFound\";i:0;s:5:\"start\";i:0;s:4:\"docs\";a:2:{i:0;a:6:{s:2:\"id\";s:1:\"1\";s:5:\"data1\";s:5:\"hello\";s:5:\"data2\";i:42;s:5:\"data3\";b:1;s:5:\"data4\";a:2:{s:7:\"data4.1\";s:7:\"hashmap\";s:7:\"data4.2\";s:5:\"hello\";}s:5:\"data5\";a:3:{i:0;s:7:\"data5.1\";i:1;s:7:\"data5.2\";i:2;s:7:\"data5.3\";}}i:1;a:1:{s:2:\"id\";s:1:\"2\";}}}}", + buf.toString()); + req.close(); + } + +} From 541a7263fa6bd9812bcdf261533eb728ad9ce0ba Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Tue, 22 Feb 2011 17:14:57 +0000 Subject: [PATCH 019/200] fix small problems with term vectors git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1073415 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/lucene/index/TermVectorsTermsWriter.java | 4 +--- .../apache/lucene/index/TermVectorsTermsWriterPerField.java | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java index 7cdcc0a550b..b79e3fc99bc 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java @@ -94,7 +94,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { // vector output files, we must abort this segment // because those files will be in an unknown // state: - hasVectors = true; tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION)); tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)); tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION)); @@ -112,7 +111,7 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument start"); - if (numVectorFields == 0) { + if (!hasVectors) { return; } @@ -201,7 +200,6 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { @Override void startDocument() throws IOException { assert clearLastVectorFieldName(); - perFields = new TermVectorsTermsWriterPerField[1]; reset(); } diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java index c27de33cd65..9f2ecc6d346 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java @@ -70,6 +70,7 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { } if (doVectors) { + termsWriter.hasVectors = true; if (termsWriter.tvx != null) { if (termsHashPerField.bytesHash.size() != 0) { // Only necessary if previous doc hit a From 05bfc5829457a4607a5828082bab18eefbc5a7b1 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Wed, 23 Feb 2011 07:29:02 +0000 Subject: [PATCH 020/200] fix silly bug that crept in during last trunk merge git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1073625 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/index/IndexWriter.java | 91 +++++++++---------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 4f0148a9840..66bfb3ed039 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -2085,12 +2085,12 @@ public class IndexWriter implements Closeable { setDiagnostics(newSegment, "flush"); - if (useCompoundFile(newSegment)) { - String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); - message("creating compound file " + compoundFileName); + boolean success = false; + try { + if (useCompoundFile(newSegment)) { + String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); + message("creating compound file " + compoundFileName); // Now build compound file - boolean success = false; - try { CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); for(String fileName : newSegment.files()) { cfsWriter.addFile(fileName); @@ -2103,56 +2103,55 @@ public class IndexWriter implements Closeable { } newSegment.setUseCompoundFile(true); + } - // Must write deleted docs after the CFS so we don't - // slurp the del file into CFS: - if (deletedDocs != null) { - final int delCount = deletedDocs.count(); - assert delCount > 0; - newSegment.setDelCount(delCount); - newSegment.advanceDelGen(); - final String delFileName = newSegment.getDelFileName(); - if (infoStream != null) { - message("flush: write " + delCount + " deletes to " + delFileName); - } - boolean success2 = false; - try { - // TODO: in the NRT case it'd be better to hand - // this del vector over to the - // shortly-to-be-opened SegmentReader and let it - // carry the changes; there's no reason to use - // filesystem as intermediary here. - deletedDocs.write(directory, delFileName); - success2 = true; - } finally { - if (!success2) { - try { - directory.deleteFile(delFileName); - } catch (Throwable t) { - // suppress this so we keep throwing the - // original exception - } - } - } + // Must write deleted docs after the CFS so we don't + // slurp the del file into CFS: + if (deletedDocs != null) { + final int delCount = deletedDocs.count(); + assert delCount > 0; + newSegment.setDelCount(delCount); + newSegment.advanceDelGen(); + final String delFileName = newSegment.getDelFileName(); + if (infoStream != null) { + message("flush: write " + delCount + " deletes to " + delFileName); } - - success = true; - } finally { - if (!success) { - if (infoStream != null) { - message("hit exception " + - "reating compound file for newly flushed segment " + newSegment.name); - } - - synchronized(this) { - deleter.refresh(newSegment.name); + boolean success2 = false; + try { + // TODO: in the NRT case it'd be better to hand + // this del vector over to the + // shortly-to-be-opened SegmentReader and let it + // carry the changes; there's no reason to use + // filesystem as intermediary here. + deletedDocs.write(directory, delFileName); + success2 = true; + } finally { + if (!success2) { + try { + directory.deleteFile(delFileName); + } catch (Throwable t) { + // suppress this so we keep throwing the + // original exception + } } } } + success = true; + } finally { + if (!success) { + if (infoStream != null) { + message("hit exception " + + "reating compound file for newly flushed segment " + newSegment.name); + } + synchronized(this) { + deleter.refresh(newSegment.name); + } + } } + synchronized(this) { segmentInfos.add(newSegment); checkpoint(); From da65d16dbe6d82d2d8d4170991e38ddb582e1137 Mon Sep 17 00:00:00 2001 From: Michael Busch Date: Fri, 25 Feb 2011 07:15:28 +0000 Subject: [PATCH 021/200] LUCENE-2324: new doc deletes approach; various bug fixes git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1074414 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/DocumentsWriter.java | 182 +++++++----------- .../index/DocumentsWriterPerThread.java | 49 ++++- .../org/apache/lucene/index/IndexWriter.java | 13 +- .../index/TermVectorsTermsWriterPerField.java | 2 - .../lucene/index/TestRollingUpdates.java | 17 +- 5 files changed, 132 insertions(+), 131 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 12c81f31526..a0b1ed91ce1 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -27,13 +27,13 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; import org.apache.lucene.search.Query; import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BitVector; /** * This class accepts multiple added documents and directly @@ -145,28 +145,24 @@ final class DocumentsWriter { } boolean deleteQueries(final Query... queries) throws IOException { + synchronized(this) { + for (Query query : queries) { + pendingDeletes.addQuery(query, BufferedDeletes.MAX_INT); + } + } + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); - boolean deleted = false; while (threadsIterator.hasNext()) { ThreadState state = threadsIterator.next(); state.lock(); try { state.perThread.deleteQueries(queries); - deleted = true; } finally { state.unlock(); } } - if (!deleted) { - synchronized(this) { - for (Query query : queries) { - pendingDeletes.addQuery(query, BufferedDeletes.MAX_INT); - } - } - } - return false; } @@ -175,12 +171,16 @@ final class DocumentsWriter { } boolean deleteTerms(final Term... terms) throws IOException { + synchronized(this) { + for (Term term : terms) { + pendingDeletes.addTerm(term, BufferedDeletes.MAX_INT); + } + } + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); - boolean deleted = false; while (threadsIterator.hasNext()) { ThreadState state = threadsIterator.next(); - deleted = true; state.lock(); try { state.perThread.deleteTerms(terms); @@ -189,14 +189,6 @@ final class DocumentsWriter { } } - if (!deleted) { - synchronized(this) { - for (Term term : terms) { - pendingDeletes.addTerm(term, BufferedDeletes.MAX_INT); - } - } - } - return false; } @@ -207,12 +199,14 @@ final class DocumentsWriter { return deleteTerms(term); } - boolean deleteTerm(final Term term, ThreadState exclude) { + void deleteTerm(final Term term, ThreadState exclude) { + synchronized(this) { + pendingDeletes.addTerm(term, BufferedDeletes.MAX_INT); + } + Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); - boolean deleted = false; while (threadsIterator.hasNext()) { - deleted = true; ThreadState state = threadsIterator.next(); if (state != exclude) { state.lock(); @@ -223,8 +217,6 @@ final class DocumentsWriter { } } } - - return deleted; } /** If non-null, various details of indexing are printed @@ -303,6 +295,10 @@ final class DocumentsWriter { synchronized void abort() throws IOException { boolean success = false; + synchronized (this) { + pendingDeletes.clear(); + } + try { if (infoStream != null) { message("docWriter: abort"); @@ -328,7 +324,7 @@ final class DocumentsWriter { } } - synchronized boolean anyChanges() { + boolean anyChanges() { return numDocsInRAM.get() != 0 || anyDeletions(); } @@ -355,29 +351,10 @@ final class DocumentsWriter { return numDeletes; } - // TODO: can we improve performance of this method by keeping track - // here in DW of whether any DWPT has deletions? - public synchronized boolean anyDeletions() { - if (pendingDeletes.any()) { - return true; + public boolean anyDeletions() { + return pendingDeletes.any(); } - Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); - while (threadsIterator.hasNext()) { - ThreadState state = threadsIterator.next(); - state.lock(); - try { - if (state.perThread.pendingDeletes.any()) { - return true; - } - } finally { - state.unlock(); - } - } - - return false; - } - void close() { closed = true; } @@ -386,9 +363,7 @@ final class DocumentsWriter { throws CorruptIndexException, IOException { ensureOpen(); - SegmentInfo newSegment = null; - BufferedDeletes segmentDeletes = null; - BitVector deletedDocs = null; + FlushedSegment newSegment = null; ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(), this, doc); try { @@ -398,39 +373,38 @@ final class DocumentsWriter { numDocsInRAM.incrementAndGet(); newSegment = finishAddDocument(dwpt, perThreadRAMUsedBeforeAdd); - if (newSegment != null) { - deletedDocs = dwpt.flushState.deletedDocs; - if (dwpt.pendingDeletes.any()) { - segmentDeletes = dwpt.pendingDeletes; - dwpt.pendingDeletes = new BufferedDeletes(false); - } - } } finally { perThread.unlock(); } - if (segmentDeletes != null) { - pushDeletes(newSegment, segmentDeletes); - } - - if (newSegment != null) { - perThreadPool.clearThreadBindings(perThread); - indexWriter.addFlushedSegment(newSegment, deletedDocs); - return true; - } - // delete term from other DWPTs later, so that this thread // doesn't have to lock multiple DWPTs at the same time if (delTerm != null) { deleteTerm(delTerm, perThread); } + if (newSegment != null) { + finishFlushedSegment(newSegment); + } + + if (newSegment != null) { + perThreadPool.clearThreadBindings(perThread); + return true; + } + return false; + } + + private void finishFlushedSegment(FlushedSegment newSegment) throws IOException { + pushDeletes(newSegment); + if (newSegment != null) { + indexWriter.addFlushedSegment(newSegment); + } } - private final SegmentInfo finishAddDocument(DocumentsWriterPerThread perThread, + private final FlushedSegment finishAddDocument(DocumentsWriterPerThread perThread, long perThreadRAMUsedBeforeAdd) throws IOException { - SegmentInfo newSegment = null; + FlushedSegment newSegment = null; if (perThread.getNumDocsInRAM() == maxBufferedDocs) { newSegment = perThread.flush(); @@ -445,20 +419,21 @@ final class DocumentsWriter { return newSegment; } - final void substractFlushedNumDocs(int numFlushed) { + final void subtractFlushedNumDocs(int numFlushed) { int oldValue = numDocsInRAM.get(); while (!numDocsInRAM.compareAndSet(oldValue, oldValue - numFlushed)) { oldValue = numDocsInRAM.get(); } } - private final void pushDeletes(SegmentInfo segmentInfo, BufferedDeletes segmentDeletes) { - synchronized(indexWriter) { - // Lock order: DW -> BD + private synchronized void pushDeletes(FlushedSegment flushedSegment) { + maybePushPendingDeletes(); + if (flushedSegment != null) { + BufferedDeletes deletes = flushedSegment.segmentDeletes; final long delGen = bufferedDeletesStream.getNextGen(); - if (segmentDeletes.any()) { - if (indexWriter.segmentInfos.size() > 0 || segmentInfo != null) { - final FrozenBufferedDeletes packet = new FrozenBufferedDeletes(segmentDeletes, delGen); + // Lock order: DW -> BD + if (deletes != null && deletes.any()) { + final FrozenBufferedDeletes packet = new FrozenBufferedDeletes(deletes, delGen); if (infoStream != null) { message("flush: push buffered deletes"); } @@ -466,40 +441,27 @@ final class DocumentsWriter { if (infoStream != null) { message("flush: delGen=" + packet.gen); } - if (segmentInfo != null) { - segmentInfo.setBufferedDeletesGen(packet.gen); } - } else { - if (infoStream != null) { - message("flush: drop buffered deletes: no segments"); + flushedSegment.segmentInfo.setBufferedDeletesGen(delGen); } - // We can safely discard these deletes: since - // there are no segments, the deletions cannot - // affect anything. } - } else if (segmentInfo != null) { - segmentInfo.setBufferedDeletesGen(delGen); + + private synchronized final void maybePushPendingDeletes() { + final long delGen = bufferedDeletesStream.getNextGen(); + if (pendingDeletes.any()) { + bufferedDeletesStream.push(new FrozenBufferedDeletes(pendingDeletes, delGen)); + pendingDeletes.clear(); } } - } final boolean flushAllThreads(final boolean flushDeletes) throws IOException { - if (flushDeletes) { - synchronized (this) { - pushDeletes(null, pendingDeletes); - pendingDeletes = new BufferedDeletes(false); - } - } - Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); boolean anythingFlushed = false; while (threadsIterator.hasNext()) { - SegmentInfo newSegment = null; - BufferedDeletes segmentDeletes = null; - BitVector deletedDocs = null; + FlushedSegment newSegment = null; ThreadState perThread = threadsIterator.next(); perThread.lock(); @@ -520,34 +482,24 @@ final class DocumentsWriter { newSegment = dwpt.flush(); if (newSegment != null) { - anythingFlushed = true; - deletedDocs = dwpt.flushState.deletedDocs; perThreadPool.clearThreadBindings(perThread); - if (dwpt.pendingDeletes.any()) { - segmentDeletes = dwpt.pendingDeletes; - dwpt.pendingDeletes = new BufferedDeletes(false); } } - } else if (flushDeletes && dwpt.pendingDeletes.any()) { - segmentDeletes = dwpt.pendingDeletes; - dwpt.pendingDeletes = new BufferedDeletes(false); - } } finally { perThread.unlock(); } - if (segmentDeletes != null) { - pushDeletes(newSegment, segmentDeletes); - } - - if (newSegment != null) { - // important do unlock the perThread before finishFlushedSegment - // is called to prevent deadlock on IndexWriter mutex - indexWriter.addFlushedSegment(newSegment, deletedDocs); + anythingFlushed = true; + finishFlushedSegment(newSegment); } } + if (!anythingFlushed && flushDeletes) { + maybePushPendingDeletes(); + } + + return anythingFlushed; } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 2614632f870..5b7e5d2b4d2 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -101,6 +101,26 @@ public class DocumentsWriterPerThread { public boolean testPoint(String name) { return docWriter.writer.testPoint(name); } + + public void clear() { + // don't hold onto doc nor analyzer, in case it is + // largish: + doc = null; + analyzer = null; + } + } + + static class FlushedSegment { + final SegmentInfo segmentInfo; + final BufferedDeletes segmentDeletes; + final BitVector deletedDocuments; + + private FlushedSegment(SegmentInfo segmentInfo, + BufferedDeletes segmentDeletes, BitVector deletedDocuments) { + this.segmentInfo = segmentInfo; + this.segmentDeletes = segmentDeletes; + this.deletedDocuments = deletedDocuments; + } } /** Called if we hit an exception at a bad time (when @@ -136,7 +156,6 @@ public class DocumentsWriterPerThread { final Directory directory; final DocState docState; final DocConsumer consumer; - private DocFieldProcessor docFieldProcessor; String segment; // Current segment we are working on boolean aborting; // True if an abort is pending @@ -160,10 +179,7 @@ public class DocumentsWriterPerThread { this.docState.similarityProvider = parent.indexWriter.getConfig().getSimilarityProvider(); consumer = indexingChain.getChain(this); - if (consumer instanceof DocFieldProcessor) { - docFieldProcessor = (DocFieldProcessor) consumer; } - } void setAborting() { aborting = true; @@ -175,7 +191,7 @@ public class DocumentsWriterPerThread { docState.analyzer = analyzer; docState.docID = numDocsInRAM; if (delTerm != null) { - pendingDeletes.addTerm(delTerm, docState.docID); + pendingDeletes.addTerm(delTerm, numDocsInRAM); } if (segment == null) { @@ -186,7 +202,11 @@ public class DocumentsWriterPerThread { boolean success = false; try { + try { consumer.processDocument(fieldInfos); + } finally { + docState.clear(); + } success = true; } finally { @@ -230,16 +250,20 @@ public class DocumentsWriterPerThread { } void deleteQueries(Query... queries) { + if (numDocsInRAM > 0) { for (Query query : queries) { pendingDeletes.addQuery(query, numDocsInRAM); } } + } void deleteTerms(Term... terms) { + if (numDocsInRAM > 0) { for (Term term : terms) { pendingDeletes.addTerm(term, numDocsInRAM); } } + } int getNumDocsInRAM() { return numDocsInRAM; @@ -254,12 +278,12 @@ public class DocumentsWriterPerThread { segment = null; consumer.doAfterFlush(); fieldInfos = fieldInfos.newFieldInfosWithGlobalFieldNumberMap(); - parent.substractFlushedNumDocs(numDocsInRAM); + parent.subtractFlushedNumDocs(numDocsInRAM); numDocsInRAM = 0; } /** Flush all pending docs to a new segment */ - SegmentInfo flush() throws IOException { + FlushedSegment flush() throws IOException { assert numDocsInRAM > 0; flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos, @@ -295,6 +319,7 @@ public class DocumentsWriterPerThread { SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentCodecs, fieldInfos); consumer.flush(flushState); + pendingDeletes.terms.clear(); newSegment.clearFilesCache(); if (infoStream != null) { @@ -305,11 +330,19 @@ public class DocumentsWriterPerThread { } flushedDocCount += flushState.numDocs; + BufferedDeletes segmentDeletes = null; + if (pendingDeletes.queries.isEmpty()) { + pendingDeletes.clear(); + } else { + segmentDeletes = pendingDeletes; + pendingDeletes = new BufferedDeletes(false); + } + doAfterFlush(); success = true; - return newSegment; + return new FlushedSegment(newSegment, segmentDeletes, flushState.deletedDocs); } finally { if (!success) { if (segment != null) { diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 66bfb3ed039..6ae23be406a 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; import org.apache.lucene.index.codecs.CodecProvider; @@ -2080,8 +2081,10 @@ public class IndexWriter implements Closeable { deleter.checkpoint(segmentInfos, false); } - void addFlushedSegment(SegmentInfo newSegment, BitVector deletedDocs) throws IOException { - assert newSegment != null; + void addFlushedSegment(FlushedSegment flushedSegment) throws IOException { + assert flushedSegment != null; + + SegmentInfo newSegment = flushedSegment.segmentInfo; setDiagnostics(newSegment, "flush"); @@ -2107,8 +2110,8 @@ public class IndexWriter implements Closeable { // Must write deleted docs after the CFS so we don't // slurp the del file into CFS: - if (deletedDocs != null) { - final int delCount = deletedDocs.count(); + if (flushedSegment.deletedDocuments != null) { + final int delCount = flushedSegment.deletedDocuments.count(); assert delCount > 0; newSegment.setDelCount(delCount); newSegment.advanceDelGen(); @@ -2123,7 +2126,7 @@ public class IndexWriter implements Closeable { // shortly-to-be-opened SegmentReader and let it // carry the changes; there's no reason to use // filesystem as intermediary here. - deletedDocs.write(directory, delFileName); + flushedSegment.deletedDocuments.write(directory, delFileName); success2 = true; } finally { if (!success2) { diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java index 9f2ecc6d346..c8f34158dfe 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java @@ -71,7 +71,6 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { if (doVectors) { termsWriter.hasVectors = true; - if (termsWriter.tvx != null) { if (termsHashPerField.bytesHash.size() != 0) { // Only necessary if previous doc hit a // non-aborting exception while writing vectors in @@ -79,7 +78,6 @@ final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField { termsHashPerField.reset(); } } - } // TODO: only if needed for performance //perThread.postingsCount = 0; diff --git a/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java b/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java index c38fd2d4b2c..a50dee8e001 100644 --- a/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java +++ b/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.*; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.*; import org.apache.lucene.util.*; import org.junit.Test; @@ -48,7 +49,21 @@ public class TestRollingUpdates extends LuceneTestCase { id++; } doc.getField("id").setValue(myID); - w.updateDocument(new Term("id", myID), doc); + int mode = docIter % 3; + switch (mode) { + case 0: { + w.deleteDocuments(new Term("id", myID)); + w.addDocument(doc); + break; + } + case 1: { + w.deleteDocuments(new TermQuery(new Term("id", myID))); + w.addDocument(doc); + break; + } + default : w.updateDocument(new Term("id", myID), doc); + } + if (docIter >= SIZE && random.nextInt(50) == 17) { if (r != null) { From c5694b06b4a6b93b5b90f84e964e476601c067c8 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Sun, 27 Feb 2011 15:23:41 +0000 Subject: [PATCH 022/200] removed @Override on implemented Interface methods git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1075076 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/index/DocumentsWriterPerThreadPool.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java index 1cde642704c..86bc1c1cdef 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java @@ -62,17 +62,14 @@ public abstract class DocumentsWriterPerThreadPool { return new Iterator() { int i = 0; - @Override public boolean hasNext() { return i < upto; } - @Override public ThreadState next() { return perThreads[i++]; } - @Override public void remove() { throw new UnsupportedOperationException("remove() not supported."); } From 6346b91b75aab99cc997bc739be4adf896d90a45 Mon Sep 17 00:00:00 2001 From: Shai Erera Date: Mon, 28 Mar 2011 11:11:23 +0000 Subject: [PATCH 023/200] rename ant-junit-LICENSE to *-ASL so that 'validate' succeeds git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086189 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/lib/{ant-junit-LICENSE.txt => ant-junit-LICENSE-ASL.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lucene/lib/{ant-junit-LICENSE.txt => ant-junit-LICENSE-ASL.txt} (100%) diff --git a/lucene/lib/ant-junit-LICENSE.txt b/lucene/lib/ant-junit-LICENSE-ASL.txt similarity index 100% rename from lucene/lib/ant-junit-LICENSE.txt rename to lucene/lib/ant-junit-LICENSE-ASL.txt From 76b494593d8d633d99daee6dc3562894b7c97bcd Mon Sep 17 00:00:00 2001 From: Koji Sekiguchi Date: Mon, 28 Mar 2011 15:39:10 +0000 Subject: [PATCH 024/200] SOLR-2445: Register standard search handler git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086276 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 28 ++++++++++++- .../org/apache/solr/core/RequestHandlers.java | 39 +++++++++++++------ 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 21c28d78961..3c211ae9d2f 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -214,7 +214,33 @@ Apache Lucene trunk Apache Tika 0.8 Carrot2 3.4.2 -(No Changes) + +Upgrading from Solr 3.1 +---------------------- + +Detailed Change List +---------------------- + +New Features +---------------------- + +Optimizations +---------------------- + +Bug Fixes +---------------------- + +* SOLR-2445: Register "standard" search handler. (koji) + +Other Changes +---------------------- + +Build +---------------------- + +Documentation +---------------------- + ================== 3.1.0-dev ================== Versions of Major Components diff --git a/solr/src/java/org/apache/solr/core/RequestHandlers.java b/solr/src/java/org/apache/solr/core/RequestHandlers.java index ddd6b3aa8f4..ef672625269 100644 --- a/solr/src/java/org/apache/solr/core/RequestHandlers.java +++ b/solr/src/java/org/apache/solr/core/RequestHandlers.java @@ -17,18 +17,6 @@ package org.apache.solr.core; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.solr.util.plugin.PluginInfoInitialized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.net.URL; import java.util.Collections; import java.util.HashMap; @@ -36,6 +24,21 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CommonParams.EchoParamStyle; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.handler.component.SearchHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.util.plugin.PluginInfoInitialized; +import org.apache.solr.util.plugin.SolrCoreAware; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** */ final class RequestHandlers { @@ -68,6 +71,18 @@ final class RequestHandlers { public RequestHandlers(SolrCore core) { this.core = core; + register(DEFAULT_HANDLER_NAME, getStandardHandler()); + } + + private SolrRequestHandler getStandardHandler(){ + SolrRequestHandler standard = core.createRequestHandler(SearchHandler.class.getName()); + NamedList defParams = new NamedList(); + defParams.add(CommonParams.HEADER_ECHO_PARAMS, EchoParamStyle.EXPLICIT.toString()); + defParams.add(CommonParams.ROWS, 10); + NamedList nl = new NamedList(); + nl.add("defaults", defParams); + standard.init(nl); + return standard; } /** From c7c03f6c61997ff24b5bc65056120e8569ef5955 Mon Sep 17 00:00:00 2001 From: Shai Erera Date: Mon, 28 Mar 2011 15:54:39 +0000 Subject: [PATCH 025/200] LUCENE-2996: addIndexes(IndexReader) does not flush before adding the new indexes, and as a consequence, deletes are applied on the incoming indexes too git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086288 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 4 +++ .../org/apache/lucene/index/IndexWriter.java | 4 +++ .../apache/lucene/index/TestAddIndexes.java | 34 +++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 4784ca1a609..e93f3cf0945 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -366,6 +366,10 @@ Bug fixes * LUCENE-2936: PhraseQuery score explanations were not correctly identifying matches vs non-matches. (hossman) +* LUCENE-2996: addIndexes(IndexReader) did not flush before adding the new + indexes, causing existing deletions to be applied on the incoming indexes as + well. (Shai Erera, Mike McCandless) + ======================= Lucene 3.x (not yet released) ======================= Changes in backwards compatibility policy diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index cedd1990905..374934bef79 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -2211,6 +2211,10 @@ public class IndexWriter implements Closeable { ensureOpen(); try { + if (infoStream != null) + message("flush at addIndexes(IndexReader...)"); + flush(false, true); + String mergedName = newSegmentName(); SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, null, codecs, payloadProcessorProvider, diff --git a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java index a0055751e43..6085d342c17 100755 --- a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -938,6 +938,40 @@ public class TestAddIndexes extends LuceneTestCase { assertTrue(c.failures.size() == 0); } + + // LUCENE-2996: tests that addIndexes(IndexReader) applies existing deletes correctly. + public void testExistingDeletes() throws Exception { + Directory[] dirs = new Directory[2]; + for (int i = 0; i < dirs.length; i++) { + dirs[i] = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()); + IndexWriter writer = new IndexWriter(dirs[i], conf); + Document doc = new Document(); + doc.add(new Field("id", "myid", Store.NO, Index.NOT_ANALYZED_NO_NORMS)); + writer.addDocument(doc); + writer.close(); + } + + IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()); + IndexWriter writer = new IndexWriter(dirs[0], conf); + + // Now delete the document + writer.deleteDocuments(new Term("id", "myid")); + IndexReader r = IndexReader.open(dirs[1]); + try { + writer.addIndexes(r); + } finally { + r.close(); + } + writer.commit(); + assertEquals("Documents from the incoming index should not have been deleted", 1, writer.numDocs()); + writer.close(); + + for (Directory dir : dirs) { + dir.close(); + } + + } private void addDocs3(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { From b822bb427ca3c56431fd56a556a5346483c038fe Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 29 Mar 2011 13:48:37 +0000 Subject: [PATCH 026/200] SOLR-2423 -- fixing DateField and adding test git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086582 13f79535-47bb-0310-9956-ffa450edef68 --- solr/src/java/org/apache/solr/schema/DateField.java | 8 ++++++++ .../test/org/apache/solr/schema/DateFieldTest.java | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/solr/src/java/org/apache/solr/schema/DateField.java b/solr/src/java/org/apache/solr/schema/DateField.java index e6888a7d697..a47aea715b4 100644 --- a/solr/src/java/org/apache/solr/schema/DateField.java +++ b/solr/src/java/org/apache/solr/schema/DateField.java @@ -180,6 +180,14 @@ public class DateField extends FieldType { "Invalid Date Math String:'" +val+'\'',e); } } + + public Fieldable createField(SchemaField field, Object value, float boost) { + // Convert to a string before indexing + if(value instanceof Date) { + value = toInternal( (Date)value ) + 'Z'; + } + return super.createField(field, value, boost); + } public String toInternal(Date val) { return formatDate(val); diff --git a/solr/src/test/org/apache/solr/schema/DateFieldTest.java b/solr/src/test/org/apache/solr/schema/DateFieldTest.java index 9168d4fe993..8cec0894f8c 100644 --- a/solr/src/test/org/apache/solr/schema/DateFieldTest.java +++ b/solr/src/test/org/apache/solr/schema/DateFieldTest.java @@ -17,6 +17,8 @@ package org.apache.solr.schema; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.schema.DateField; import org.apache.solr.util.DateMathParser; @@ -114,4 +116,13 @@ public class DateFieldTest extends LuceneTestCase { } + public void testCreateField() { + int props = FieldProperties.INDEXED ^ FieldProperties.STORED; + SchemaField sf = new SchemaField( "test", f, props, null ); + Fieldable out = (Field)f.createField(sf, "1995-12-31T23:59:59Z", 1.0f ); + assertEquals(820454399000l, f.toObject( out ).getTime() ); + + out = (Field)f.createField(sf, new Date(820454399000l), 1.0f ); + assertEquals(820454399000l, f.toObject( out ).getTime() ); + } } From 1c5398f0e087d6fd404dd1fe9ad696a5e122eaf7 Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Tue, 29 Mar 2011 13:52:14 +0000 Subject: [PATCH 027/200] LUCENE-2998: forward port prepare-release and stage git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086584 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/build.xml | 5 ++++- solr/build.xml | 4 ++-- solr/common-build.xml | 7 ++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lucene/build.xml b/lucene/build.xml index 4d245d595f2..a35460e9be2 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -424,9 +424,12 @@ - + + + + diff --git a/solr/build.xml b/solr/build.xml index c19ab981fff..3c3c6090556 100644 --- a/solr/build.xml +++ b/solr/build.xml @@ -934,14 +934,14 @@ keyfile="${keyfile}" verbose="true" > - + - diff --git a/solr/common-build.xml b/solr/common-build.xml index 36ca6f0194d..420f75726e4 100644 --- a/solr/common-build.xml +++ b/solr/common-build.xml @@ -414,6 +414,8 @@ + + @@ -421,10 +423,13 @@ Signing @{input.file} Sig File: @{output.file} - + + + + From 5da11b6bff38f9943d62b37f1dfa86897b200029 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Tue, 29 Mar 2011 15:33:11 +0000 Subject: [PATCH 028/200] silly: use the right random var git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086628 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java b/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java index 12ee8d8539a..41b64e0f891 100644 --- a/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java +++ b/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java @@ -191,7 +191,7 @@ public class TestFSTs extends LuceneTestCase { } final char[] buffer = new char[end]; for (int i = 0; i < end; i++) { - buffer[i] = (char) _TestUtil.nextInt(random, 97, 102); + buffer[i] = (char) _TestUtil.nextInt(r, 97, 102); } return new String(buffer, 0, end); } From 447b497c577b5ff8875574157bce2f2dee63ebb8 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Tue, 29 Mar 2011 15:33:45 +0000 Subject: [PATCH 029/200] use SMS when testing LogMP's strict invariants git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086629 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene/index/TestIndexWriterMergePolicy.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java index 8b05149e3bd..299abceceda 100755 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java @@ -131,7 +131,8 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase { IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()) - .setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())); + .setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy()) + .setMergeScheduler(new SerialMergeScheduler())); // leftmost* segment has 1 doc // rightmost* segment has 100 docs @@ -144,8 +145,8 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase { writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode( - OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy( - new LogDocMergePolicy())); + OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy()) + .setMergeScheduler(new SerialMergeScheduler())); } writer.close(); @@ -153,7 +154,7 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase { ldmp.setMergeFactor(10); writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode( - OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler())); + OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new SerialMergeScheduler())); // merge policy only fixes segments on levels where merges // have been triggered, so check invariants after all adds @@ -236,13 +237,13 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase { int segmentCount = writer.getSegmentCount(); for (int i = segmentCount - 1; i >= 0; i--) { int docCount = writer.getDocCount(i); - assertTrue("docCount=" + docCount + " lowerBound=" + lowerBound + " i=" + i + " segmentCount=" + segmentCount + " index=" + writer.segString(), docCount > lowerBound); + assertTrue("docCount=" + docCount + " lowerBound=" + lowerBound + " upperBound=" + upperBound + " i=" + i + " segmentCount=" + segmentCount + " index=" + writer.segString() + " config=" + writer.getConfig(), docCount > lowerBound); if (docCount <= upperBound) { numSegments++; } else { if (upperBound * mergeFactor <= maxMergeDocs) { - assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString(), numSegments < mergeFactor); + assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString() + " config=" + writer.getConfig(), numSegments < mergeFactor); } do { From 4bfb56d42bf1b93e9e2bd1bec3ed9f7bbbf5543c Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 29 Mar 2011 16:37:39 +0000 Subject: [PATCH 030/200] SOLR-2396: add [ICU]CollationField git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086637 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 4 + solr/contrib/analysis-extras/CHANGES.txt | 4 +- .../ICUCollationKeyFilterFactory.java | 2 + .../apache/solr/schema/ICUCollationField.java | 228 ++++++++++++++++ .../analysis-extras/src/test-files/empty | 4 - .../conf/schema-icucollate.xml | 59 +++++ .../conf/solrconfig-icucollate.xml | 23 ++ .../TestICUCollationKeyFilterFactory.java | 1 + .../solr/schema/TestICUCollationField.java | 186 +++++++++++++ .../analysis/CollationKeyFilterFactory.java | 2 + .../apache/solr/schema/CollationField.java | 250 ++++++++++++++++++ .../apache/solr/search/SolrQueryParser.java | 6 +- .../test-files/solr/conf/schema-collate.xml | 62 +++++ .../solr/conf/solrconfig-collate.xml | 23 ++ .../solr/schema/TestCollationField.java | 198 ++++++++++++++ 15 files changed, 1044 insertions(+), 8 deletions(-) create mode 100644 solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java delete mode 100644 solr/contrib/analysis-extras/src/test-files/empty create mode 100644 solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/schema-icucollate.xml create mode 100644 solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/solrconfig-icucollate.xml create mode 100644 solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java create mode 100644 solr/src/java/org/apache/solr/schema/CollationField.java create mode 100644 solr/src/test-files/solr/conf/schema-collate.xml create mode 100644 solr/src/test-files/solr/conf/solrconfig-collate.xml create mode 100644 solr/src/test/org/apache/solr/schema/TestCollationField.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 3c211ae9d2f..1ebf1ffa49c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -113,6 +113,10 @@ New Features as results. (ryan with patches from grant, noble, cmale, yonik) * SOLR-2417: Add explain info directly to return documents using ?fl=_explain_ (ryan) + +* SOLR-2396: Add CollationField, which is much more efficient than + the Solr 3.x CollationKeyFilterFactory, and also supports + Locale-sensitive range queries. (rmuir) Optimizations diff --git a/solr/contrib/analysis-extras/CHANGES.txt b/solr/contrib/analysis-extras/CHANGES.txt index 6f26f9f6683..72053f89d8b 100644 --- a/solr/contrib/analysis-extras/CHANGES.txt +++ b/solr/contrib/analysis-extras/CHANGES.txt @@ -13,7 +13,9 @@ analyzers for Chinese and Polish. $Id$ ================== Release 4.0-dev ================== -(No Changes) +* SOLR-2396: Add ICUCollationField, which is much more efficient than + the Solr 3.x ICUCollationKeyFilterFactory, and also supports + Locale-sensitive range queries. (rmuir) ================== Release 3.2-dev ================== diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUCollationKeyFilterFactory.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUCollationKeyFilterFactory.java index 1a79de899f6..e88a8f0128e 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUCollationKeyFilterFactory.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/ICUCollationKeyFilterFactory.java @@ -57,7 +57,9 @@ import com.ibm.icu.util.ULocale; * @see Collator * @see ULocale * @see RuleBasedCollator + * @deprecated use {@link org.apache.solr.schema.ICUCollationField} instead. */ +@Deprecated public class ICUCollationKeyFilterFactory extends BaseTokenFilterFactory implements ResourceLoaderAware { private Collator collator; diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java new file mode 100644 index 00000000000..dbbbfc08aaa --- /dev/null +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java @@ -0,0 +1,228 @@ +package org.apache.solr.schema; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; +import org.apache.lucene.collation.ICUCollationKeyAnalyzer; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.apache.solr.common.ResourceLoader; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.QParser; + +import com.ibm.icu.text.Collator; +import com.ibm.icu.text.RuleBasedCollator; +import com.ibm.icu.util.ULocale; + +/** + * Field for collated sort keys. + * These can be used for locale-sensitive sort and range queries. + *

+ * This field can be created in two ways: + *

    + *
  • Based upon a system collator associated with a Locale. + *
  • Based upon a tailored ruleset. + *
+ *

+ * Using a System collator: + *

    + *
  • locale: RFC 3066 locale ID (mandatory) + *
  • strength: 'primary','secondary','tertiary', 'quaternary', or 'identical' (optional) + *
  • decomposition: 'no', or 'canonical' (optional) + *
+ *

+ * Using a Tailored ruleset: + *

    + *
  • custom: UTF-8 text file containing rules supported by RuleBasedCollator (mandatory) + *
  • strength: 'primary','secondary','tertiary', 'quaternary', or 'identical' (optional) + *
  • decomposition: 'no' or 'canonical' (optional) + *
+ * + * @see Collator + * @see ULocale + * @see RuleBasedCollator + */ +public class ICUCollationField extends FieldType { + private Analyzer analyzer; + + @Override + protected void init(IndexSchema schema, Map args) { + properties |= TOKENIZED; // this ensures our analyzer gets hit + setup(schema.getResourceLoader(), args); + super.init(schema, args); + } + + /** + * Setup the field according to the provided parameters + */ + private void setup(ResourceLoader loader, Map args) { + String custom = args.remove("custom"); + String localeID = args.remove("locale"); + String strength = args.remove("strength"); + String decomposition = args.remove("decomposition"); + + if (custom == null && localeID == null) + throw new SolrException(ErrorCode.SERVER_ERROR, "Either custom or locale is required."); + + if (custom != null && localeID != null) + throw new SolrException(ErrorCode.SERVER_ERROR, "Cannot specify both locale and custom. " + + "To tailor rules for a built-in language, see the javadocs for RuleBasedCollator. " + + "Then save the entire customized ruleset to a file, and use with the custom parameter"); + + final Collator collator; + + if (localeID != null) { + // create from a system collator, based on Locale. + collator = createFromLocale(localeID); + } else { + // create from a custom ruleset + collator = createFromRules(custom, loader); + } + + // set the strength flag, otherwise it will be the default. + if (strength != null) { + if (strength.equalsIgnoreCase("primary")) + collator.setStrength(Collator.PRIMARY); + else if (strength.equalsIgnoreCase("secondary")) + collator.setStrength(Collator.SECONDARY); + else if (strength.equalsIgnoreCase("tertiary")) + collator.setStrength(Collator.TERTIARY); + else if (strength.equalsIgnoreCase("quaternary")) + collator.setStrength(Collator.QUATERNARY); + else if (strength.equalsIgnoreCase("identical")) + collator.setStrength(Collator.IDENTICAL); + else + throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid strength: " + strength); + } + + // set the decomposition flag, otherwise it will be the default. + if (decomposition != null) { + if (decomposition.equalsIgnoreCase("no")) + collator.setDecomposition(Collator.NO_DECOMPOSITION); + else if (decomposition.equalsIgnoreCase("canonical")) + collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION); + else + throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid decomposition: " + decomposition); + } + // we use 4.0 because it ensures we just encode the pure byte[] keys. + analyzer = new ICUCollationKeyAnalyzer(Version.LUCENE_40, collator); + } + + /** + * Create a locale from localeID. + * Then return the appropriate collator for the locale. + */ + private Collator createFromLocale(String localeID) { + return Collator.getInstance(new ULocale(localeID)); + } + + /** + * Read custom rules from a file, and create a RuleBasedCollator + * The file cannot support comments, as # might be in the rules! + */ + private Collator createFromRules(String fileName, ResourceLoader loader) { + InputStream input = null; + try { + input = loader.openResource(fileName); + String rules = IOUtils.toString(input, "UTF-8"); + return new RuleBasedCollator(rules); + } catch (Exception e) { + // io error or invalid rules + throw new RuntimeException(e); + } finally { + IOUtils.closeQuietly(input); + } + } + + @Override + public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException { + writer.writeStr(name, f.stringValue(), true); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + return getStringSort(field, top); + } + + @Override + public Analyzer getAnalyzer() { + return analyzer; + } + + @Override + public Analyzer getQueryAnalyzer() { + return analyzer; + } + + /** + * analyze the range with the analyzer, instead of the collator. + * because icu collators are not thread safe, this keeps things + * simple (we already have a threadlocal clone in the reused TS) + */ + private BytesRef analyzeRangePart(String field, String part) { + TokenStream source; + + try { + source = analyzer.reusableTokenStream(field, new StringReader(part)); + source.reset(); + } catch (IOException e) { + source = analyzer.tokenStream(field, new StringReader(part)); + } + + TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); + BytesRef bytes = termAtt.getBytesRef(); + + // we control the analyzer here: most errors are impossible + try { + if (!source.incrementToken()) + throw new IllegalArgumentException("analyzer returned no terms for range part: " + part); + termAtt.fillBytesRef(); + assert !source.incrementToken(); + } catch (IOException e) { + throw new RuntimeException("error analyzing range part: " + part, e); + } + + try { + source.close(); + } catch (IOException ignored) {} + + return new BytesRef(bytes); + } + + @Override + public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) { + String f = field.getName(); + BytesRef low = part1 == null ? null : analyzeRangePart(f, part1); + BytesRef high = part2 == null ? null : analyzeRangePart(f, part2); + return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive); + } +} diff --git a/solr/contrib/analysis-extras/src/test-files/empty b/solr/contrib/analysis-extras/src/test-files/empty deleted file mode 100644 index ad1d7bb4e6a..00000000000 --- a/solr/contrib/analysis-extras/src/test-files/empty +++ /dev/null @@ -1,4 +0,0 @@ -Please don't remove this silly file! - -This is here to make sure the dir is not empty... otherwise Ht/git -clones have problems. diff --git a/solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/schema-icucollate.xml b/solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/schema-icucollate.xml new file mode 100644 index 00000000000..3ec19c65175 --- /dev/null +++ b/solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/schema-icucollate.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + diff --git a/solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/solrconfig-icucollate.xml b/solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/solrconfig-icucollate.xml new file mode 100644 index 00000000000..2c9b55c1a6a --- /dev/null +++ b/solr/contrib/analysis-extras/src/test-files/solr-analysis-extras/conf/solrconfig-icucollate.xml @@ -0,0 +1,23 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java index 44c42f6f2f6..e99105276e3 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java @@ -34,6 +34,7 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; +@Deprecated public class TestICUCollationKeyFilterFactory extends BaseTokenTestCase { /* diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java new file mode 100644 index 00000000000..ddf9d0f4f95 --- /dev/null +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.io.File; +import java.io.FileOutputStream; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.BeforeClass; + +import com.ibm.icu.text.Collator; +import com.ibm.icu.text.RuleBasedCollator; +import com.ibm.icu.util.ULocale; + +/** + * Tests {@link ICUCollationField} with TermQueries, RangeQueries, and sort order. + */ +public class TestICUCollationField extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + String home = setupSolrHome(); + initCore("solrconfig.xml","schema.xml", home); + // add some docs + assertU(adoc("id", "1", "text", "\u0633\u0627\u0628")); + assertU(adoc("id", "2", "text", "I WİLL USE TURKİSH CASING")); + assertU(adoc("id", "3", "text", "ı will use turkish casıng")); + assertU(adoc("id", "4", "text", "Töne")); + assertU(adoc("id", "5", "text", "I W\u0049\u0307LL USE TURKİSH CASING")); + assertU(adoc("id", "6", "text", "Ｔｅｓｔｉｎｇ")); + assertU(adoc("id", "7", "text", "Tone")); + assertU(adoc("id", "8", "text", "Testing")); + assertU(adoc("id", "9", "text", "testing")); + assertU(adoc("id", "10", "text", "toene")); + assertU(adoc("id", "11", "text", "Tzne")); + assertU(adoc("id", "12", "text", "\u0698\u0698")); + assertU(commit()); + } + + /** + * Ugly: but what to do? We want to test custom sort, which reads rules in as a resource. + * These are largish files, and jvm-specific (as our documentation says, you should always + * look out for jvm differences with collation). + * So its preferable to create this file on-the-fly. + */ + public static String setupSolrHome() throws Exception { + // make a solr home underneath the test's TEMP_DIR + File tmpFile = File.createTempFile("test", "tmp", TEMP_DIR); + tmpFile.delete(); + tmpFile.mkdir(); + + // make data and conf dirs + new File(tmpFile, "data").mkdir(); + File confDir = new File(tmpFile, "conf"); + confDir.mkdir(); + + // copy over configuration files + FileUtils.copyFile(getFile("solr-analysis-extras/conf/solrconfig-icucollate.xml"), new File(confDir, "solrconfig.xml")); + FileUtils.copyFile(getFile("solr-analysis-extras/conf/schema-icucollate.xml"), new File(confDir, "schema.xml")); + + // generate custom collation rules (DIN 5007-2), saving to customrules.dat + RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new ULocale("de", "DE")); + + String DIN5007_2_tailorings = + "& ae , a\u0308 & AE , A\u0308"+ + "& oe , o\u0308 & OE , O\u0308"+ + "& ue , u\u0308 & UE , u\u0308"; + + RuleBasedCollator tailoredCollator = new RuleBasedCollator(baseCollator.getRules() + DIN5007_2_tailorings); + String tailoredRules = tailoredCollator.getRules(); + FileOutputStream os = new FileOutputStream(new File(confDir, "customrules.dat")); + IOUtils.write(tailoredRules, os, "UTF-8"); + os.close(); + + return tmpFile.getAbsolutePath(); + } + + /** + * Test termquery with german DIN 5007-1 primary strength. + * In this case, ö is equivalent to o (but not oe) + */ + public void testBasicTermQuery() { + assertQ("Collated TQ: ", + req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=4]", + "//result/doc[2]/int[@name='id'][.=7]" + ); + } + + /** + * Test rangequery again with the DIN 5007-1 collator. + * We do a range query of tone .. tp, in binary order this + * would retrieve nothing due to case and accent differences. + */ + public void testBasicRangeQuery() { + assertQ("Collated RangeQ: ", + req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=4]", + "//result/doc[2]/int[@name='id'][.=7]" + ); + } + + /** + * Test sort with a danish collator. ö is ordered after z + */ + public void testBasicSort() { + assertQ("Collated Sort: ", + req("fl", "id", "q", "sort_da:[tz TO töz]", "sort", "sort_da asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=11]", + "//result/doc[2]/int[@name='id'][.=4]" + ); + } + + /** + * Test sort with an arabic collator. U+0633 is ordered after U+0698. + * With a binary collator, the range would also return nothing. + */ + public void testArabicSort() { + assertQ("Collated Sort: ", + req("fl", "id", "q", "sort_ar:[\u0698 TO \u0633\u0633]", "sort", "sort_ar asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=12]", + "//result/doc[2]/int[@name='id'][.=1]" + ); + } + + /** + * Test rangequery again with an Arabic collator. + * Binary order would normally order U+0633 in this range. + */ + public void testNegativeRangeQuery() { + assertQ("Collated RangeQ: ", + req("fl", "id", "q", "sort_ar:[\u062F TO \u0698]", "sort", "id asc" ), + "//*[@numFound='0']" + ); + } + /** + * Test canonical decomposition with turkish primary strength. + * With this sort order, İ is the uppercase form of i, and I is the uppercase form of ı. + * We index a decomposed form of İ. + */ + public void testCanonicalDecomposition() { + assertQ("Collated TQ: ", + req("fl", "id", "q", "sort_tr_canon:\"I Will Use Turkish Casıng\"", "sort", "id asc" ), + "//*[@numFound='3']", + "//result/doc[1]/int[@name='id'][.=2]", + "//result/doc[2]/int[@name='id'][.=3]", + "//result/doc[3]/int[@name='id'][.=5]" + ); + } + + /** + * Test termquery with custom collator (DIN 5007-2). + * In this case, ö is equivalent to oe (but not o) + */ + public void testCustomCollation() { + assertQ("Collated TQ: ", + req("fl", "id", "q", "sort_custom:toene", "sort", "id asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=4]", + "//result/doc[2]/int[@name='id'][.=10]" + ); + } +} diff --git a/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java b/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java index c342d36e0bf..b8de5e52410 100644 --- a/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java +++ b/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java @@ -69,7 +69,9 @@ import org.apache.solr.util.plugin.ResourceLoaderAware; * @see Locale * @see RuleBasedCollator * @since solr 3.1 + * @deprecated use {@link org.apache.solr.schema.CollationField} instead. */ +@Deprecated public class CollationKeyFilterFactory extends BaseTokenFilterFactory implements ResourceLoaderAware { private Collator collator; diff --git a/solr/src/java/org/apache/solr/schema/CollationField.java b/solr/src/java/org/apache/solr/schema/CollationField.java new file mode 100644 index 00000000000..7f915a4724a --- /dev/null +++ b/solr/src/java/org/apache/solr/schema/CollationField.java @@ -0,0 +1,250 @@ +package org.apache.solr.schema; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.text.Collator; +import java.text.ParseException; +import java.text.RuleBasedCollator; +import java.util.Locale; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; +import org.apache.lucene.collation.CollationKeyAnalyzer; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.apache.solr.common.ResourceLoader; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.QParser; + +/** + * Field for collated sort keys. + * These can be used for locale-sensitive sort and range queries. + *

+ * This field can be created in two ways: + *

    + *
  • Based upon a system collator associated with a Locale. + *
  • Based upon a tailored ruleset. + *
+ *

+ * Using a System collator: + *

    + *
  • language: ISO-639 language code (mandatory) + *
  • country: ISO-3166 country code (optional) + *
  • variant: vendor or browser-specific code (optional) + *
  • strength: 'primary','secondary','tertiary', or 'identical' (optional) + *
  • decomposition: 'no','canonical', or 'full' (optional) + *
+ *

+ * Using a Tailored ruleset: + *

    + *
  • custom: UTF-8 text file containing rules supported by RuleBasedCollator (mandatory) + *
  • strength: 'primary','secondary','tertiary', or 'identical' (optional) + *
  • decomposition: 'no','canonical', or 'full' (optional) + *
+ * + * @see Collator + * @see Locale + * @see RuleBasedCollator + * @since solr 4.0 + */ +public class CollationField extends FieldType { + private Analyzer analyzer; + + @Override + protected void init(IndexSchema schema, Map args) { + properties |= TOKENIZED; // this ensures our analyzer gets hit + setup(schema.getResourceLoader(), args); + super.init(schema, args); + } + + /** + * Setup the field according to the provided parameters + */ + private void setup(ResourceLoader loader, Map args) { + String custom = args.remove("custom"); + String language = args.remove("language"); + String country = args.remove("country"); + String variant = args.remove("variant"); + String strength = args.remove("strength"); + String decomposition = args.remove("decomposition"); + + final Collator collator; + + if (custom == null && language == null) + throw new SolrException(ErrorCode.SERVER_ERROR, "Either custom or language is required."); + + if (custom != null && + (language != null || country != null || variant != null)) + throw new SolrException(ErrorCode.SERVER_ERROR, "Cannot specify both language and custom. " + + "To tailor rules for a built-in language, see the javadocs for RuleBasedCollator. " + + "Then save the entire customized ruleset to a file, and use with the custom parameter"); + + if (language != null) { + // create from a system collator, based on Locale. + collator = createFromLocale(language, country, variant); + } else { + // create from a custom ruleset + collator = createFromRules(custom, loader); + } + + // set the strength flag, otherwise it will be the default. + if (strength != null) { + if (strength.equalsIgnoreCase("primary")) + collator.setStrength(Collator.PRIMARY); + else if (strength.equalsIgnoreCase("secondary")) + collator.setStrength(Collator.SECONDARY); + else if (strength.equalsIgnoreCase("tertiary")) + collator.setStrength(Collator.TERTIARY); + else if (strength.equalsIgnoreCase("identical")) + collator.setStrength(Collator.IDENTICAL); + else + throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid strength: " + strength); + } + + // set the decomposition flag, otherwise it will be the default. + if (decomposition != null) { + if (decomposition.equalsIgnoreCase("no")) + collator.setDecomposition(Collator.NO_DECOMPOSITION); + else if (decomposition.equalsIgnoreCase("canonical")) + collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION); + else if (decomposition.equalsIgnoreCase("full")) + collator.setDecomposition(Collator.FULL_DECOMPOSITION); + else + throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid decomposition: " + decomposition); + } + // we use 4.0 because it ensures we just encode the pure byte[] keys. + analyzer = new CollationKeyAnalyzer(Version.LUCENE_40, collator); + } + + /** + * Create a locale from language, with optional country and variant. + * Then return the appropriate collator for the locale. + */ + private Collator createFromLocale(String language, String country, String variant) { + Locale locale; + + if (language != null && country == null && variant != null) + throw new SolrException(ErrorCode.SERVER_ERROR, + "To specify variant, country is required"); + else if (language != null && country != null && variant != null) + locale = new Locale(language, country, variant); + else if (language != null && country != null) + locale = new Locale(language, country); + else + locale = new Locale(language); + + return Collator.getInstance(locale); + } + + /** + * Read custom rules from a file, and create a RuleBasedCollator + * The file cannot support comments, as # might be in the rules! + */ + private Collator createFromRules(String fileName, ResourceLoader loader) { + InputStream input = null; + try { + input = loader.openResource(fileName); + String rules = IOUtils.toString(input, "UTF-8"); + return new RuleBasedCollator(rules); + } catch (IOException e) { + // io error + throw new RuntimeException(e); + } catch (ParseException e) { + // invalid rules + throw new RuntimeException(e); + } finally { + IOUtils.closeQuietly(input); + } + } + + @Override + public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException { + writer.writeStr(name, f.stringValue(), true); + } + + @Override + public SortField getSortField(SchemaField field, boolean top) { + return getStringSort(field, top); + } + + @Override + public Analyzer getAnalyzer() { + return analyzer; + } + + @Override + public Analyzer getQueryAnalyzer() { + return analyzer; + } + + /** + * analyze the range with the analyzer, instead of the collator. + * because jdk collators might not be thread safe (when they are + * its just that all methods are synced), this keeps things + * simple (we already have a threadlocal clone in the reused TS) + */ + private BytesRef analyzeRangePart(String field, String part) { + TokenStream source; + + try { + source = analyzer.reusableTokenStream(field, new StringReader(part)); + source.reset(); + } catch (IOException e) { + source = analyzer.tokenStream(field, new StringReader(part)); + } + + TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); + BytesRef bytes = termAtt.getBytesRef(); + + // we control the analyzer here: most errors are impossible + try { + if (!source.incrementToken()) + throw new IllegalArgumentException("analyzer returned no terms for range part: " + part); + termAtt.fillBytesRef(); + assert !source.incrementToken(); + } catch (IOException e) { + throw new RuntimeException("error analyzing range part: " + part, e); + } + + try { + source.close(); + } catch (IOException ignored) {} + + return new BytesRef(bytes); + } + + @Override + public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) { + String f = field.getName(); + BytesRef low = part1 == null ? null : analyzeRangePart(f, part1); + BytesRef high = part2 == null ? null : analyzeRangePart(f, part2); + return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive); + } +} diff --git a/solr/src/java/org/apache/solr/search/SolrQueryParser.java b/solr/src/java/org/apache/solr/search/SolrQueryParser.java index d0df3423673..34192b8fd8b 100644 --- a/solr/src/java/org/apache/solr/search/SolrQueryParser.java +++ b/solr/src/java/org/apache/solr/search/SolrQueryParser.java @@ -122,9 +122,9 @@ public class SolrQueryParser extends QueryParser { SchemaField sf = schema.getFieldOrNull(field); if (sf != null) { FieldType ft = sf.getType(); - // delegate to type for everything except TextField - if (ft instanceof TextField) { - return super.getFieldQuery(field, queryText, quoted || ((TextField)ft).getAutoGeneratePhraseQueries()); + // delegate to type for everything except tokenized fields + if (ft.isTokenized()) { + return super.getFieldQuery(field, queryText, quoted || (ft instanceof TextField && ((TextField)ft).getAutoGeneratePhraseQueries())); } else { return sf.getType().getFieldQuery(parser, sf, queryText); } diff --git a/solr/src/test-files/solr/conf/schema-collate.xml b/solr/src/test-files/solr/conf/schema-collate.xml new file mode 100644 index 00000000000..7feb73a3015 --- /dev/null +++ b/solr/src/test-files/solr/conf/schema-collate.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + diff --git a/solr/src/test-files/solr/conf/solrconfig-collate.xml b/solr/src/test-files/solr/conf/solrconfig-collate.xml new file mode 100644 index 00000000000..2c9b55c1a6a --- /dev/null +++ b/solr/src/test-files/solr/conf/solrconfig-collate.xml @@ -0,0 +1,23 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + diff --git a/solr/src/test/org/apache/solr/schema/TestCollationField.java b/solr/src/test/org/apache/solr/schema/TestCollationField.java new file mode 100644 index 00000000000..48ad341792c --- /dev/null +++ b/solr/src/test/org/apache/solr/schema/TestCollationField.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.io.File; +import java.io.FileOutputStream; +import java.text.Collator; +import java.text.RuleBasedCollator; +import java.util.Locale; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.BeforeClass; + +/** + * Tests {@link CollationField} with TermQueries, RangeQueries, and sort order. + */ +public class TestCollationField extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + String home = setupSolrHome(); + initCore("solrconfig.xml","schema.xml", home); + // add some docs + assertU(adoc("id", "1", "text", "\u0633\u0627\u0628")); + assertU(adoc("id", "2", "text", "I WİLL USE TURKİSH CASING")); + assertU(adoc("id", "3", "text", "ı will use turkish casıng")); + assertU(adoc("id", "4", "text", "Töne")); + assertU(adoc("id", "5", "text", "I W\u0049\u0307LL USE TURKİSH CASING")); + assertU(adoc("id", "6", "text", "Ｔｅｓｔｉｎｇ")); + assertU(adoc("id", "7", "text", "Tone")); + assertU(adoc("id", "8", "text", "Testing")); + assertU(adoc("id", "9", "text", "testing")); + assertU(adoc("id", "10", "text", "toene")); + assertU(adoc("id", "11", "text", "Tzne")); + assertU(adoc("id", "12", "text", "\u0698\u0698")); + assertU(commit()); + } + + /** + * Ugly: but what to do? We want to test custom sort, which reads rules in as a resource. + * These are largish files, and jvm-specific (as our documentation says, you should always + * look out for jvm differences with collation). + * So its preferable to create this file on-the-fly. + */ + public static String setupSolrHome() throws Exception { + // make a solr home underneath the test's TEMP_DIR + File tmpFile = File.createTempFile("test", "tmp", TEMP_DIR); + tmpFile.delete(); + tmpFile.mkdir(); + + // make data and conf dirs + new File(tmpFile, "data").mkdir(); + File confDir = new File(tmpFile, "conf"); + confDir.mkdir(); + + // copy over configuration files + FileUtils.copyFile(getFile("solr/conf/solrconfig-collate.xml"), new File(confDir, "solrconfig.xml")); + FileUtils.copyFile(getFile("solr/conf/schema-collate.xml"), new File(confDir, "schema.xml")); + + // generate custom collation rules (DIN 5007-2), saving to customrules.dat + RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new Locale("de", "DE")); + + String DIN5007_2_tailorings = + "& ae , a\u0308 & AE , A\u0308"+ + "& oe , o\u0308 & OE , O\u0308"+ + "& ue , u\u0308 & UE , u\u0308"; + + RuleBasedCollator tailoredCollator = new RuleBasedCollator(baseCollator.getRules() + DIN5007_2_tailorings); + String tailoredRules = tailoredCollator.getRules(); + FileOutputStream os = new FileOutputStream(new File(confDir, "customrules.dat")); + IOUtils.write(tailoredRules, os, "UTF-8"); + os.close(); + + return tmpFile.getAbsolutePath(); + } + + /** + * Test termquery with german DIN 5007-1 primary strength. + * In this case, ö is equivalent to o (but not oe) + */ + public void testBasicTermQuery() { + assertQ("Collated TQ: ", + req("fl", "id", "q", "sort_de:tone", "sort", "id asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=4]", + "//result/doc[2]/int[@name='id'][.=7]" + ); + } + + /** + * Test rangequery again with the DIN 5007-1 collator. + * We do a range query of tone .. tp, in binary order this + * would retrieve nothing due to case and accent differences. + */ + public void testBasicRangeQuery() { + assertQ("Collated RangeQ: ", + req("fl", "id", "q", "sort_de:[tone TO tp]", "sort", "id asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=4]", + "//result/doc[2]/int[@name='id'][.=7]" + ); + } + + /** + * Test sort with a danish collator. ö is ordered after z + */ + public void testBasicSort() { + assertQ("Collated Sort: ", + req("fl", "id", "q", "sort_da:[tz TO töz]", "sort", "sort_da asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=11]", + "//result/doc[2]/int[@name='id'][.=4]" + ); + } + + /** + * Test sort with an arabic collator. U+0633 is ordered after U+0698. + * With a binary collator, the range would also return nothing. + */ + public void testArabicSort() { + assertQ("Collated Sort: ", + req("fl", "id", "q", "sort_ar:[\u0698 TO \u0633\u0633]", "sort", "sort_ar asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=12]", + "//result/doc[2]/int[@name='id'][.=1]" + ); + } + + /** + * Test rangequery again with an Arabic collator. + * Binary order would normally order U+0633 in this range. + */ + public void testNegativeRangeQuery() { + assertQ("Collated RangeQ: ", + req("fl", "id", "q", "sort_ar:[\u062F TO \u0698]", "sort", "id asc" ), + "//*[@numFound='0']" + ); + } + /** + * Test canonical decomposition with turkish primary strength. + * With this sort order, İ is the uppercase form of i, and I is the uppercase form of ı. + * We index a decomposed form of İ. + */ + public void testCanonicalDecomposition() { + assertQ("Collated TQ: ", + req("fl", "id", "q", "sort_tr_canon:\"I Will Use Turkish Casıng\"", "sort", "id asc" ), + "//*[@numFound='3']", + "//result/doc[1]/int[@name='id'][.=2]", + "//result/doc[2]/int[@name='id'][.=3]", + "//result/doc[3]/int[@name='id'][.=5]" + ); + } + + /** + * Test full decomposition with chinese identical strength. + * The full width form "Ｔｅｓｔｉｎｇ" is treated identical to "Testing" + */ + public void testFullDecomposition() { + assertQ("Collated TQ: ", + req("fl", "id", "q", "sort_zh_full:Testing", "sort", "id asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=6]", + "//result/doc[2]/int[@name='id'][.=8]" + ); + } + + /** + * Test termquery with custom collator (DIN 5007-2). + * In this case, ö is equivalent to oe (but not o) + */ + public void testCustomCollation() { + assertQ("Collated TQ: ", + req("fl", "id", "q", "sort_custom:toene", "sort", "id asc" ), + "//*[@numFound='2']", + "//result/doc[1]/int[@name='id'][.=4]", + "//result/doc[2]/int[@name='id'][.=10]" + ); + } +} From b95ca2b4539e024aac5983dafdaa1a1a61f75d43 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 29 Mar 2011 17:41:58 +0000 Subject: [PATCH 031/200] LUCENE-3001 -- Adding TrieFieldHelper to write solr compatible numeric fields without the solr dependency. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086651 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 4 + .../apache/lucene/util/TrieFieldHelper.java | 163 ++++++++++++++++++ .../org/apache/solr/schema/TrieDateField.java | 48 +----- .../org/apache/solr/schema/TrieField.java | 148 ++++------------ 4 files changed, 209 insertions(+), 154 deletions(-) create mode 100644 lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index e93f3cf0945..448f11b0707 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -333,6 +333,10 @@ New features * LUCENE-2862: Added TermsEnum.totalTermFreq() and Terms.getSumTotalTermFreq(). (Mike McCandless, Robert Muir) + +* LUCENE-3001: Added TrieFieldHelper to write solr compatible numeric + fields without the solr dependency. (ryan) + Optimizations diff --git a/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java b/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java new file mode 100644 index 00000000000..e16090af198 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java @@ -0,0 +1,163 @@ +/** + * Copyright 2005 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.util; + +import java.util.Date; + +import org.apache.lucene.analysis.NumericTokenStream; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; + +/** + * Helper class to make TrieFields compatible with ones written in solr + */ +public class TrieFieldHelper { + + private TrieFieldHelper() {} + + public static class FieldInfo { + public int precisionStep = 8; // same as solr default + public boolean store = true; + public boolean index = true; + public boolean omitNorms = true; + public boolean omitTF = true; + } + + //---------------------------------------------- + // Create Field + //---------------------------------------------- + + private static Fieldable createField(String name, byte[] arr, TokenStream ts, FieldInfo info, float boost) { + + Field f; + if (info.store) { + f = new Field(name, arr); + if (info.index) f.setTokenStream(ts); + } else { + f = new Field(name, ts); + } + + // term vectors aren't supported + f.setOmitNorms(info.omitNorms); + f.setOmitTermFreqAndPositions(info.omitTF); + f.setBoost(boost); + return f; + } + + public static Fieldable createIntField(String name, int value, FieldInfo info, float boost) { + + byte[] arr=null; + TokenStream ts=null; + + if (info.store) arr = TrieFieldHelper.toArr(value); + if (info.index) ts = new NumericTokenStream(info.precisionStep).setIntValue(value); + + return createField(name, arr, ts, info, boost); + } + + public static Fieldable createFloatField(String name, float value, FieldInfo info, float boost) { + + byte[] arr=null; + TokenStream ts=null; + + if (info.store) arr = TrieFieldHelper.toArr(value); + if (info.index) ts = new NumericTokenStream(info.precisionStep).setFloatValue(value); + + return createField(name, arr, ts, info, boost); + } + + public static Fieldable createLongField(String name, long value, FieldInfo info, float boost) { + + byte[] arr=null; + TokenStream ts=null; + + if (info.store) arr = TrieFieldHelper.toArr(value); + if (info.index) ts = new NumericTokenStream(info.precisionStep).setLongValue(value); + + return createField(name, arr, ts, info, boost); + } + + public static Fieldable createDoubleField(String name, double value, FieldInfo info, float boost) { + + byte[] arr=null; + TokenStream ts=null; + + if (info.store) arr = TrieFieldHelper.toArr(value); + if (info.index) ts = new NumericTokenStream(info.precisionStep).setDoubleValue(value); + + return createField(name, arr, ts, info, boost); + } + + public static Fieldable createDateField(String name, Date value, FieldInfo info, float boost) { + // TODO, make sure the date is within long range! + return createLongField(name, value.getTime(), info, boost); + } + + + //---------------------------------------------- + // number <=> byte[] + //---------------------------------------------- + + public static int toInt(byte[] arr) { + return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff); + } + + public static long toLong(byte[] arr) { + int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff); + int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff); + return (((long)high)<<32) | (low&0x0ffffffffL); + } + + public static float toFloat(byte[] arr) { + return Float.intBitsToFloat(toInt(arr)); + } + + public static double toDouble(byte[] arr) { + return Double.longBitsToDouble(toLong(arr)); + } + + public static byte[] toArr(int val) { + byte[] arr = new byte[4]; + arr[0] = (byte)(val>>>24); + arr[1] = (byte)(val>>>16); + arr[2] = (byte)(val>>>8); + arr[3] = (byte)(val); + return arr; + } + + public static byte[] toArr(long val) { + byte[] arr = new byte[8]; + arr[0] = (byte)(val>>>56); + arr[1] = (byte)(val>>>48); + arr[2] = (byte)(val>>>40); + arr[3] = (byte)(val>>>32); + arr[4] = (byte)(val>>>24); + arr[5] = (byte)(val>>>16); + arr[6] = (byte)(val>>>8); + arr[7] = (byte)(val); + return arr; + } + + public static byte[] toArr(float val) { + return toArr(Float.floatToRawIntBits(val)); + } + + public static byte[] toArr(double val) { + return toArr(Double.doubleToRawLongBits(val)); + } +} diff --git a/solr/src/java/org/apache/solr/schema/TrieDateField.java b/solr/src/java/org/apache/solr/schema/TrieDateField.java index bba4d0ae69e..f02d26311a1 100755 --- a/solr/src/java/org/apache/solr/schema/TrieDateField.java +++ b/solr/src/java/org/apache/solr/schema/TrieDateField.java @@ -27,7 +27,6 @@ import org.apache.solr.search.function.*; import org.apache.solr.search.QParser; import org.apache.solr.response.TextResponseWriter; import org.apache.lucene.document.Fieldable; -import org.apache.lucene.document.Field; import org.apache.lucene.search.SortField; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.Query; @@ -36,8 +35,7 @@ import org.apache.lucene.search.cache.CachedArrayCreator; import org.apache.lucene.search.cache.LongValuesCreator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.NumericTokenStream; +import org.apache.lucene.util.TrieFieldHelper; import java.util.Map; import java.util.Date; @@ -68,7 +66,7 @@ public class TrieDateField extends DateField { public Date toObject(Fieldable f) { byte[] arr = f.getBinaryValue(); if (arr==null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,TrieField.badFieldString(f)); - return new Date(TrieField.toLong(arr)); + return new Date(TrieFieldHelper.toLong(arr)); } @Override @@ -107,7 +105,7 @@ public class TrieDateField extends DateField { return; } - writer.writeDate(name,new Date(TrieField.toLong(arr))); + writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr))); } @Override @@ -146,7 +144,7 @@ public class TrieDateField extends DateField { public String toExternal(Fieldable f) { byte[] arr = f.getBinaryValue(); if (arr==null) return TrieField.badFieldString(f); - return super.toExternal(new Date(TrieField.toLong(arr))); + return super.toExternal(new Date(TrieFieldHelper.toLong(arr))); } @Override @@ -167,44 +165,6 @@ public class TrieDateField extends DateField { return readableToIndexed(storedToReadable(f)); } - @Override - public Fieldable createField(SchemaField field, Object value, float boost) { - boolean indexed = field.indexed(); - boolean stored = field.stored(); - - if (!indexed && !stored) { - if (log.isTraceEnabled()) - log.trace("Ignoring unindexed/unstored field: " + field); - return null; - } - - int ps = precisionStep; - - byte[] arr=null; - TokenStream ts=null; - - long time = (value instanceof Date) - ? ((Date)value).getTime() - : super.parseMath(null, value.toString()).getTime(); - - if (stored) arr = TrieField.toArr(time); - if (indexed) ts = new NumericTokenStream(ps).setLongValue(time); - - Field f; - if (stored) { - f = new Field(field.getName(), arr); - if (indexed) f.setTokenStream(ts); - } else { - f = new Field(field.getName(), ts); - } - - // term vectors aren't supported - - f.setOmitNorms(field.omitNorms()); - f.setOmitTermFreqAndPositions(field.omitTf()); - f.setBoost(boost); - return f; - } @Override public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) { diff --git a/solr/src/java/org/apache/solr/schema/TrieField.java b/solr/src/java/org/apache/solr/schema/TrieField.java index cd7d953c4ad..8e7dff3bcbc 100644 --- a/solr/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/src/java/org/apache/solr/schema/TrieField.java @@ -17,7 +17,6 @@ package org.apache.solr.schema; import org.apache.lucene.document.Fieldable; -import org.apache.lucene.document.Field; import org.apache.lucene.search.*; import org.apache.lucene.search.cache.CachedArrayCreator; import org.apache.lucene.search.cache.DoubleValuesCreator; @@ -26,8 +25,7 @@ import org.apache.lucene.search.cache.IntValuesCreator; import org.apache.lucene.search.cache.LongValuesCreator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.NumericTokenStream; +import org.apache.lucene.util.TrieFieldHelper; import org.apache.noggit.CharArr; import org.apache.solr.analysis.*; import org.apache.solr.common.SolrException; @@ -68,6 +66,7 @@ public class TrieField extends FieldType { protected TrieTypes type; protected Object missingValue; + /** * Used for handling date types following the same semantics as DateField */ @@ -107,15 +106,15 @@ public class TrieField extends FieldType { if (arr==null) return badFieldString(f); switch (type) { case INTEGER: - return toInt(arr); + return TrieFieldHelper.toInt(arr); case FLOAT: - return toFloat(arr); + return TrieFieldHelper.toFloat(arr); case LONG: - return toLong(arr); + return TrieFieldHelper.toLong(arr); case DOUBLE: - return toDouble(arr); + return TrieFieldHelper.toDouble(arr); case DATE: - return new Date(toLong(arr)); + return new Date(TrieFieldHelper.toLong(arr)); default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); } @@ -207,19 +206,19 @@ public class TrieField extends FieldType { } switch (type) { case INTEGER: - writer.writeInt(name,toInt(arr)); + writer.writeInt(name,TrieFieldHelper.toInt(arr)); break; case FLOAT: - writer.writeFloat(name,toFloat(arr)); + writer.writeFloat(name,TrieFieldHelper.toFloat(arr)); break; case LONG: - writer.writeLong(name,toLong(arr)); + writer.writeLong(name,TrieFieldHelper.toLong(arr)); break; case DOUBLE: - writer.writeDouble(name,toDouble(arr)); + writer.writeDouble(name,TrieFieldHelper.toDouble(arr)); break; case DATE: - writer.writeDate(name,new Date(toLong(arr))); + writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr))); break; default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); @@ -293,55 +292,6 @@ public class TrieField extends FieldType { } - static int toInt(byte[] arr) { - return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff); - } - - static long toLong(byte[] arr) { - int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff); - int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff); - return (((long)high)<<32) | (low&0x0ffffffffL); - } - - static float toFloat(byte[] arr) { - return Float.intBitsToFloat(toInt(arr)); - } - - static double toDouble(byte[] arr) { - return Double.longBitsToDouble(toLong(arr)); - } - - static byte[] toArr(int val) { - byte[] arr = new byte[4]; - arr[0] = (byte)(val>>>24); - arr[1] = (byte)(val>>>16); - arr[2] = (byte)(val>>>8); - arr[3] = (byte)(val); - return arr; - } - - static byte[] toArr(long val) { - byte[] arr = new byte[8]; - arr[0] = (byte)(val>>>56); - arr[1] = (byte)(val>>>48); - arr[2] = (byte)(val>>>40); - arr[3] = (byte)(val>>>32); - arr[4] = (byte)(val>>>24); - arr[5] = (byte)(val>>>16); - arr[6] = (byte)(val>>>8); - arr[7] = (byte)(val); - return arr; - } - - static byte[] toArr(float val) { - return toArr(Float.floatToRawIntBits(val)); - } - - static byte[] toArr(double val) { - return toArr(Double.doubleToRawLongBits(val)); - } - - @Override public String storedToReadable(Fieldable f) { return toExternal(f); @@ -396,15 +346,15 @@ public class TrieField extends FieldType { if (arr==null) return badFieldString(f); switch (type) { case INTEGER: - return Integer.toString(toInt(arr)); + return Integer.toString(TrieFieldHelper.toInt(arr)); case FLOAT: - return Float.toString(toFloat(arr)); + return Float.toString(TrieFieldHelper.toFloat(arr)); case LONG: - return Long.toString(toLong(arr)); + return Long.toString(TrieFieldHelper.toLong(arr)); case DOUBLE: - return Double.toString(toDouble(arr)); + return Double.toString(TrieFieldHelper.toDouble(arr)); case DATE: - return dateField.formatDate(new Date(toLong(arr))); + return dateField.formatDate(new Date(TrieFieldHelper.toLong(arr))); default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); } @@ -483,75 +433,53 @@ public class TrieField extends FieldType { @Override public Fieldable createField(SchemaField field, Object value, float boost) { - boolean indexed = field.indexed(); - boolean stored = field.stored(); - - if (!indexed && !stored) { + TrieFieldHelper.FieldInfo info = new TrieFieldHelper.FieldInfo(); + info.index = field.indexed(); + info.store = field.stored(); + info.precisionStep = precisionStep; + info.omitNorms = field.omitNorms(); + info.omitTF = field.omitTf(); + + if (!info.index && !info.store) { if (log.isTraceEnabled()) log.trace("Ignoring unindexed/unstored field: " + field); return null; } - int ps = precisionStep; - - byte[] arr=null; - TokenStream ts=null; - // String indexedVal = indexed && precisionStep==0 ? readableToIndexed(externalVal) : null; - switch (type) { case INTEGER: int i = (value instanceof Number) ? ((Number)value).intValue() : Integer.parseInt(value.toString()); - if (stored) arr = toArr(i); - if (indexed) ts = new NumericTokenStream(ps).setIntValue(i); - break; + return TrieFieldHelper.createIntField(field.getName(), i, info, boost); + case FLOAT: float f = (value instanceof Number) ? ((Number)value).floatValue() : Float.parseFloat(value.toString()); - if (stored) arr = toArr(f); - if (indexed) ts = new NumericTokenStream(ps).setFloatValue(f); - break; + return TrieFieldHelper.createFloatField(field.getName(), f, info, boost); + case LONG: long l = (value instanceof Number) ? ((Number)value).longValue() : Long.parseLong(value.toString()); - if (stored) arr = toArr(l); - if (indexed) ts = new NumericTokenStream(ps).setLongValue(l); - break; + return TrieFieldHelper.createLongField(field.getName(), l, info, boost); + case DOUBLE: double d = (value instanceof Number) ? ((Number)value).doubleValue() : Double.parseDouble(value.toString()); - if (stored) arr = toArr(d); - if (indexed) ts = new NumericTokenStream(ps).setDoubleValue(d); - break; + return TrieFieldHelper.createDoubleField(field.getName(), d, info, boost); + case DATE: - long time = (value instanceof Date) - ? ((Date)value).getTime() - : dateField.parseMath(null, value.toString()).getTime(); - if (stored) arr = toArr(time); - if (indexed) ts = new NumericTokenStream(ps).setLongValue(time); - break; + Date date = (value instanceof Date) + ? ((Date)value) + : dateField.parseMath(null, value.toString()); + return TrieFieldHelper.createDateField(field.getName(), date, info, boost); + default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type); } - - Field f; - if (stored) { - f = new Field(field.getName(), arr); - if (indexed) f.setTokenStream(ts); - } else { - f = new Field(field.getName(), ts); - } - - // term vectors aren't supported - - f.setOmitNorms(field.omitNorms()); - f.setOmitTermFreqAndPositions(field.omitTf()); - f.setBoost(boost); - return f; } public enum TrieTypes { From 9027875384bddf95740bf6cd0712154482b7b53f Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 29 Mar 2011 18:45:54 +0000 Subject: [PATCH 032/200] LUCENE-3001 fix solr trie date field from #1086651 git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086681 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/solr/schema/TrieDateField.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/solr/src/java/org/apache/solr/schema/TrieDateField.java b/solr/src/java/org/apache/solr/schema/TrieDateField.java index f02d26311a1..12c83d500de 100755 --- a/solr/src/java/org/apache/solr/schema/TrieDateField.java +++ b/solr/src/java/org/apache/solr/schema/TrieDateField.java @@ -27,6 +27,7 @@ import org.apache.solr.search.function.*; import org.apache.solr.search.QParser; import org.apache.solr.response.TextResponseWriter; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.document.Field; import org.apache.lucene.search.SortField; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.Query; @@ -36,6 +37,8 @@ import org.apache.lucene.search.cache.LongValuesCreator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.TrieFieldHelper; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.NumericTokenStream; import java.util.Map; import java.util.Date; @@ -165,6 +168,44 @@ public class TrieDateField extends DateField { return readableToIndexed(storedToReadable(f)); } + @Override + public Fieldable createField(SchemaField field, Object value, float boost) { + boolean indexed = field.indexed(); + boolean stored = field.stored(); + + if (!indexed && !stored) { + if (log.isTraceEnabled()) + log.trace("Ignoring unindexed/unstored field: " + field); + return null; + } + + int ps = precisionStep; + + byte[] arr=null; + TokenStream ts=null; + + long time = (value instanceof Date) + ? ((Date)value).getTime() + : super.parseMath(null, value.toString()).getTime(); + + if (stored) arr = TrieFieldHelper.toArr(time); + if (indexed) ts = new NumericTokenStream(ps).setLongValue(time); + + Field f; + if (stored) { + f = new Field(field.getName(), arr); + if (indexed) f.setTokenStream(ts); + } else { + f = new Field(field.getName(), ts); + } + + // term vectors aren't supported + + f.setOmitNorms(field.omitNorms()); + f.setOmitTermFreqAndPositions(field.omitTf()); + f.setBoost(boost); + return f; + } @Override public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) { From f6ddf2723a8b218974d278a94fb9fe8f63c4cf8e Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 29 Mar 2011 19:40:37 +0000 Subject: [PATCH 033/200] SOLR-1566 -- passing SolrQueryRequest to DocTransformer git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086699 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/solr/response/transform/DocIdAugmenterFactory.java | 3 ++- .../solr/response/transform/ExplainAugmenterFactory.java | 3 ++- .../apache/solr/response/transform/ShardAugmenterFactory.java | 4 +++- .../apache/solr/response/transform/TransformerFactory.java | 4 +++- .../apache/solr/response/transform/ValueAugmenterFactory.java | 3 ++- solr/src/java/org/apache/solr/search/ReturnFields.java | 3 +-- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java b/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java index 47456d826f4..c4660f82b1d 100644 --- a/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java +++ b/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java @@ -19,6 +19,7 @@ package org.apache.solr.response.transform; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.request.SolrQueryRequest; /** * @version $Id$ @@ -27,7 +28,7 @@ import org.apache.solr.common.SolrException.ErrorCode; public class DocIdAugmenterFactory extends TransformerFactory { @Override - public DocTransformer create(String field, String arg) { + public DocTransformer create(String field, String arg, SolrQueryRequest req) { if( arg != null ) { throw new SolrException( ErrorCode.BAD_REQUEST, "DocIdAugmenter does not take any arguments" ); diff --git a/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java b/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java index 197ac96ea4b..60b2986f2ac 100644 --- a/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java +++ b/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java @@ -23,6 +23,7 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.util.SolrPluginUtils; /** @@ -62,7 +63,7 @@ public class ExplainAugmenterFactory extends TransformerFactory } @Override - public DocTransformer create(String field, String arg) { + public DocTransformer create(String field, String arg, SolrQueryRequest req) { Style style = (arg==null)?defaultStyle:getStyle(arg); return new ExplainAugmenter( field, style ); } diff --git a/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java b/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java index 0b28758ca20..388b1442d69 100644 --- a/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java +++ b/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java @@ -16,6 +16,8 @@ */ package org.apache.solr.response.transform; +import org.apache.solr.request.SolrQueryRequest; + /** * @version $Id$ @@ -24,7 +26,7 @@ package org.apache.solr.response.transform; public class ShardAugmenterFactory extends TransformerFactory { @Override - public DocTransformer create(String field, String arg) { + public DocTransformer create(String field, String arg, SolrQueryRequest req) { String id = "TODO... find ID"; // Maybe it is stored in the context? // is it a request variable? diff --git a/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java b/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java index 60531958b93..6468023b371 100644 --- a/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java +++ b/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java @@ -20,7 +20,9 @@ package org.apache.solr.response.transform; import java.util.HashMap; import java.util.Map; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.util.plugin.NamedListInitializedPlugin; /** @@ -36,7 +38,7 @@ public abstract class TransformerFactory implements NamedListInitializedPlugin defaultUserArgs = (String)args.get( "args" ); } - public abstract DocTransformer create(String field, String args); + public abstract DocTransformer create(String field, String args, SolrQueryRequest req); public static final Map defaultFactories = new HashMap(); static { diff --git a/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java b/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java index 33e7b7d933a..c2903b02fe7 100644 --- a/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java +++ b/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java @@ -21,6 +21,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.DateUtil; import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; /** * @version $Id$ @@ -60,7 +61,7 @@ public class ValueAugmenterFactory extends TransformerFactory } @Override - public DocTransformer create(String field, String arg) { + public DocTransformer create(String field, String arg, SolrQueryRequest req) { Object val = value; if( val == null ) { val = (arg==null)?defaultValue:getObjectFrom(arg); diff --git a/solr/src/java/org/apache/solr/search/ReturnFields.java b/solr/src/java/org/apache/solr/search/ReturnFields.java index 3dafb793f41..d50b417754c 100644 --- a/solr/src/java/org/apache/solr/search/ReturnFields.java +++ b/solr/src/java/org/apache/solr/search/ReturnFields.java @@ -27,7 +27,6 @@ import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.search.Query; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; @@ -315,7 +314,7 @@ public class ReturnFields TransformerFactory factory = req.getCore().getTransformerFactory( name ); if( factory != null ) { - augmenters.addTransformer( factory.create(disp, args) ); + augmenters.addTransformer( factory.create(disp, args, req) ); } else { // unknown field? From 591f3b5ea99107c7357cae74708512db52458148 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 29 Mar 2011 20:09:59 +0000 Subject: [PATCH 034/200] LUCENE-3001 -- moving TrieFieldHelper out of lucene git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086708 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/java/org/apache/solr/schema}/TrieFieldHelper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) rename {lucene/src/java/org/apache/lucene/util => solr/src/java/org/apache/solr/schema}/TrieFieldHelper.java (97%) diff --git a/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java b/solr/src/java/org/apache/solr/schema/TrieFieldHelper.java similarity index 97% rename from lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java rename to solr/src/java/org/apache/solr/schema/TrieFieldHelper.java index e16090af198..c40ecd87a78 100644 --- a/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java +++ b/solr/src/java/org/apache/solr/schema/TrieFieldHelper.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.lucene.util; +package org.apache.solr.schema; import java.util.Date; @@ -25,6 +25,9 @@ import org.apache.lucene.document.Fieldable; /** * Helper class to make TrieFields compatible with ones written in solr + * + * TODO -- Something like this should be in in lucene + * see: LUCENE-3001 */ public class TrieFieldHelper { From 9d1854b39069df05506bbe1ac748e30fbe33a72f Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Tue, 29 Mar 2011 20:11:00 +0000 Subject: [PATCH 035/200] LUCENE-3001 -- moving TrieFieldHelper out of lucene git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086710 13f79535-47bb-0310-9956-ffa450edef68 --- solr/src/java/org/apache/solr/schema/TrieDateField.java | 1 - solr/src/java/org/apache/solr/schema/TrieField.java | 1 - 2 files changed, 2 deletions(-) diff --git a/solr/src/java/org/apache/solr/schema/TrieDateField.java b/solr/src/java/org/apache/solr/schema/TrieDateField.java index 12c83d500de..7e3b30d240f 100755 --- a/solr/src/java/org/apache/solr/schema/TrieDateField.java +++ b/solr/src/java/org/apache/solr/schema/TrieDateField.java @@ -36,7 +36,6 @@ import org.apache.lucene.search.cache.CachedArrayCreator; import org.apache.lucene.search.cache.LongValuesCreator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.TrieFieldHelper; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.NumericTokenStream; diff --git a/solr/src/java/org/apache/solr/schema/TrieField.java b/solr/src/java/org/apache/solr/schema/TrieField.java index 8e7dff3bcbc..4163767cd78 100644 --- a/solr/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/src/java/org/apache/solr/schema/TrieField.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.cache.IntValuesCreator; import org.apache.lucene.search.cache.LongValuesCreator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.TrieFieldHelper; import org.apache.noggit.CharArr; import org.apache.solr.analysis.*; import org.apache.solr.common.SolrException; From 787b6019bf1d9af045314af3463a4e5760bf4628 Mon Sep 17 00:00:00 2001 From: Koji Sekiguchi Date: Wed, 30 Mar 2011 02:17:31 +0000 Subject: [PATCH 036/200] SOLR-2445: remove standard request handler and change the default qt to blank in form.jsp git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1086821 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 ++- .../java/org/apache/solr/core/RequestHandlers.java | 12 ------------ solr/src/webapp/web/admin/form.jsp | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 1ebf1ffa49c..d6e9b20d24c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -234,7 +234,8 @@ Optimizations Bug Fixes ---------------------- -* SOLR-2445: Register "standard" search handler. (koji) +* SOLR-2445: Change the default qt to blank in form.jsp, because there is no "standard" + request handler unless you have it in your solrconfig.xml explicitly. (koji) Other Changes ---------------------- diff --git a/solr/src/java/org/apache/solr/core/RequestHandlers.java b/solr/src/java/org/apache/solr/core/RequestHandlers.java index ef672625269..9541dc32043 100644 --- a/solr/src/java/org/apache/solr/core/RequestHandlers.java +++ b/solr/src/java/org/apache/solr/core/RequestHandlers.java @@ -71,18 +71,6 @@ final class RequestHandlers { public RequestHandlers(SolrCore core) { this.core = core; - register(DEFAULT_HANDLER_NAME, getStandardHandler()); - } - - private SolrRequestHandler getStandardHandler(){ - SolrRequestHandler standard = core.createRequestHandler(SearchHandler.class.getName()); - NamedList defParams = new NamedList(); - defParams.add(CommonParams.HEADER_ECHO_PARAMS, EchoParamStyle.EXPLICIT.toString()); - defParams.add(CommonParams.ROWS, 10); - NamedList nl = new NamedList(); - nl.add("defaults", defParams); - standard.init(nl); - return standard; } /** diff --git a/solr/src/webapp/web/admin/form.jsp b/solr/src/webapp/web/admin/form.jsp index 8c5e14fd29c..90dcb0aeb2f 100644 --- a/solr/src/webapp/web/admin/form.jsp +++ b/solr/src/webapp/web/admin/form.jsp @@ -72,7 +72,7 @@ Query Type - + From cf37cec0c6ab82b20055d7c889c7433c27070e63 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 30 Mar 2011 13:30:07 +0000 Subject: [PATCH 037/200] LUCENE-2573: Tiered flushing of DWPTs by RAM with low/high water marks git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/realtime_search@1086947 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/BufferedDeletes.java | 9 +- .../apache/lucene/index/DocumentsWriter.java | 284 ++++++----- .../index/DocumentsWriterFlushControl.java | 263 ++++++++++ .../index/DocumentsWriterPerThread.java | 118 +++-- .../index/DocumentsWriterPerThreadPool.java | 139 +++++- .../index/FlushByRamOrCountsPolicy.java | 66 +++ .../org/apache/lucene/index/FlushPolicy.java | 191 ++++++++ .../org/apache/lucene/index/Healthiness.java | 120 +++++ .../org/apache/lucene/index/IndexWriter.java | 209 ++------ .../lucene/index/IndexWriterConfig.java | 87 +++- .../org/apache/lucene/index/IntBlockPool.java | 6 + .../org/apache/lucene/index/TermsHash.java | 5 +- .../lucene/index/TermsHashPerField.java | 6 +- ...readAffinityDocumentsWriterThreadPool.java | 42 +- .../lucene/store/MockDirectoryWrapper.java | 9 +- .../lucene/util/ThrottledIndexOutput.java | 147 ++++++ .../index/TestFlushByRamOrCountsPolicy.java | 458 ++++++++++++++++++ .../apache/lucene/index/TestIndexWriter.java | 3 +- .../lucene/index/TestIndexWriterConfig.java | 21 + .../lucene/index/TestIndexWriterDelete.java | 2 +- 20 files changed, 1823 insertions(+), 362 deletions(-) create mode 100644 lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java create mode 100644 lucene/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java create mode 100644 lucene/src/java/org/apache/lucene/index/FlushPolicy.java create mode 100644 lucene/src/java/org/apache/lucene/index/Healthiness.java create mode 100644 lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java create mode 100644 lucene/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java diff --git a/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java b/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java index c72a1f6b0a3..ae544cbaf86 100644 --- a/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java +++ b/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java @@ -72,13 +72,18 @@ class BufferedDeletes { public static final Integer MAX_INT = Integer.valueOf(Integer.MAX_VALUE); - final AtomicLong bytesUsed = new AtomicLong(); + final AtomicLong bytesUsed; private final static boolean VERBOSE_DELETES = false; long gen; - public BufferedDeletes(boolean sortTerms) { + this(sortTerms, new AtomicLong()); + } + + BufferedDeletes(boolean sortTerms, AtomicLong bytesUsed) { + assert bytesUsed != null; + this.bytesUsed = bytesUsed; if (sortTerms) { terms = new TreeMap(); } else { diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 09d33292023..0cdd2e336a4 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; @@ -104,10 +103,8 @@ import org.apache.lucene.store.Directory; */ final class DocumentsWriter { - final AtomicLong bytesUsed = new AtomicLong(0); Directory directory; - boolean bufferIsFull; // True when it's time to write segment private volatile boolean closed; PrintStream infoStream; @@ -118,25 +115,36 @@ final class DocumentsWriter { final IndexWriter indexWriter; private AtomicInteger numDocsInRAM = new AtomicInteger(0); - private AtomicLong ramUsed = new AtomicLong(0); final BufferedDeletesStream bufferedDeletesStream; // TODO: cutover to BytesRefHash - private BufferedDeletes pendingDeletes = new BufferedDeletes(false); + private final BufferedDeletes pendingDeletes = new BufferedDeletes(false); final IndexingChain chain; - private final IndexWriterConfig config; final DocumentsWriterPerThreadPool perThreadPool; + final FlushPolicy flushPolicy; + final DocumentsWriterFlushControl flushControl; + final Healthiness healthiness; DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers, BufferedDeletesStream bufferedDeletesStream) throws IOException { this.directory = directory; this.indexWriter = writer; - this.similarityProvider = writer.getConfig().getSimilarityProvider(); + this.similarityProvider = config.getSimilarityProvider(); this.bufferedDeletesStream = bufferedDeletesStream; this.perThreadPool = config.getIndexerThreadPool(); this.chain = config.getIndexingChain(); this.perThreadPool.initialize(this, globalFieldNumbers, config); - this.config = config; + final FlushPolicy configuredPolicy = config.getFlushPolicy(); + if (configuredPolicy == null) { + flushPolicy = new FlushByRamOrCountsPolicy(); + } else { + flushPolicy = configuredPolicy; + } + flushPolicy.init(this); + + healthiness = new Healthiness(); + final long maxRamPerDWPT = config.getRAMPerThreadHardLimitMB() * 1024 * 1024; + flushControl = new DocumentsWriterFlushControl(flushPolicy, perThreadPool, healthiness, pendingDeletes, maxRamPerDWPT); } boolean deleteQueries(final Query... queries) throws IOException { @@ -146,13 +154,15 @@ final class DocumentsWriter { } } - Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + final Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); while (threadsIterator.hasNext()) { ThreadState state = threadsIterator.next(); state.lock(); try { - state.perThread.deleteQueries(queries); + if (state.isActive()) { + state.perThread.deleteQueries(queries); + } } finally { state.unlock(); } @@ -178,12 +188,17 @@ final class DocumentsWriter { ThreadState state = threadsIterator.next(); state.lock(); try { - state.perThread.deleteTerms(terms); + if (state.isActive()) { + state.perThread.deleteTerms(terms); + flushControl.doOnDelete(state); + } } finally { state.unlock(); } } - + if (flushControl.flushDeletes.getAndSet(false)) { + flushDeletes(); + } return false; } @@ -194,7 +209,7 @@ final class DocumentsWriter { return deleteTerms(term); } - void deleteTerm(final Term term, ThreadState exclude) { + void deleteTerm(final Term term, ThreadState exclude) throws IOException { synchronized(this) { pendingDeletes.addTerm(term, BufferedDeletes.MAX_INT); } @@ -207,11 +222,21 @@ final class DocumentsWriter { state.lock(); try { state.perThread.deleteTerms(term); + flushControl.doOnDelete(state); } finally { state.unlock(); } } } + if (flushControl.flushDeletes.getAndSet(false)) { + flushDeletes(); + } + } + + private void flushDeletes() throws IOException { + maybePushPendingDeletes(); + indexWriter.applyAllDeletes(); + indexWriter.flushCount.incrementAndGet(); } /** If non-null, various details of indexing are printed @@ -221,11 +246,6 @@ final class DocumentsWriter { pushConfigChange(); } - synchronized void setSimilarityProvider(SimilarityProvider similarityProvider) { - this.similarityProvider = similarityProvider; - pushConfigChange(); - } - private final void pushConfigChange() { Iterator it = perThreadPool.getAllPerThreadsIterator(); while (it.hasNext()) { @@ -245,9 +265,11 @@ final class DocumentsWriter { return abortedFiles; } - void message(String message) { + // returns boolean for asserts + boolean message(String message) { if (infoStream != null) indexWriter.message("DW: " + message); + return true; } private void ensureOpen() throws AlreadyClosedException { @@ -272,13 +294,18 @@ final class DocumentsWriter { message("docWriter: abort"); } - Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + final Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); while (threadsIterator.hasNext()) { ThreadState perThread = threadsIterator.next(); perThread.lock(); try { - perThread.perThread.abort(); + if (perThread.isActive()) { // we might be closed + perThread.perThread.abort(); + perThread.perThread.checkAndResetHasAborted(); + } else { + assert closed; + } } finally { perThread.unlock(); } @@ -297,26 +324,12 @@ final class DocumentsWriter { } public int getBufferedDeleteTermsSize() { - int size = 0; - Iterator it = perThreadPool.getActivePerThreadsIterator(); - while (it.hasNext()) { - DocumentsWriterPerThread dwpt = it.next().perThread; - size += dwpt.pendingDeletes.terms.size(); - } - size += pendingDeletes.terms.size(); - return size; + return pendingDeletes.terms.size(); } //for testing public int getNumBufferedDeleteTerms() { - int numDeletes = 0; - Iterator it = perThreadPool.getActivePerThreadsIterator(); - while (it.hasNext()) { - DocumentsWriterPerThread dwpt = it.next().perThread; - numDeletes += dwpt.pendingDeletes.numTermDeletes.get(); - } - numDeletes += pendingDeletes.numTermDeletes.get(); - return numDeletes; + return pendingDeletes.numTermDeletes.get(); } public boolean anyDeletions() { @@ -325,67 +338,89 @@ final class DocumentsWriter { void close() { closed = true; + flushControl.setClosed(); } - boolean updateDocument(final Document doc, final Analyzer analyzer, final Term delTerm) - throws CorruptIndexException, IOException { + boolean updateDocument(final Document doc, final Analyzer analyzer, + final Term delTerm) throws CorruptIndexException, IOException { ensureOpen(); - - FlushedSegment newSegment = null; - - ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(), this, doc); + boolean maybeMerge = false; + final boolean isUpdate = delTerm != null; + if (healthiness.isStalled()) { + /* + * if we are allowed to hijack threads for flushing we try to flush out + * as many pending DWPT to release memory and get back healthy status. + */ + if (infoStream != null) { + message("WARNING DocumentsWriter is stalled try to hijack thread to flush pending segment"); + } + // try pick up pending threads here if possile + final DocumentsWriterPerThread flushingDWPT; + flushingDWPT = flushControl.getFlushIfPending(null); + // don't push the delete here since the update could fail! + maybeMerge = doFlush(flushingDWPT); + if (infoStream != null && healthiness.isStalled()) { + message("WARNING DocumentsWriter is stalled might block thread until DocumentsWriter is not stalled anymore"); + } + healthiness.waitIfStalled(); // block if stalled + } + ThreadState perThread = perThreadPool.getAndLock(Thread.currentThread(), + this, doc); + DocumentsWriterPerThread flushingDWPT = null; try { - DocumentsWriterPerThread dwpt = perThread.perThread; - long perThreadRAMUsedBeforeAdd = dwpt.bytesUsed(); - dwpt.updateDocument(doc, analyzer, delTerm); + if (!perThread.isActive()) { + ensureOpen(); + assert false: "perThread is not active but we are still open"; + } + final DocumentsWriterPerThread dwpt = perThread.perThread; + try { + dwpt.updateDocument(doc, analyzer, delTerm); + } finally { + if(dwpt.checkAndResetHasAborted()) { + flushControl.doOnAbort(perThread); + } + } + flushingDWPT = flushControl.doAfterDocument(perThread, isUpdate); numDocsInRAM.incrementAndGet(); - - newSegment = finishAddDocument(dwpt, perThreadRAMUsedBeforeAdd); } finally { perThread.unlock(); } - // delete term from other DWPTs later, so that this thread // doesn't have to lock multiple DWPTs at the same time - if (delTerm != null) { + if (isUpdate) { deleteTerm(delTerm, perThread); } + maybeMerge |= doFlush(flushingDWPT); + return maybeMerge; + } + + - if (newSegment != null) { - finishFlushedSegment(newSegment); + private boolean doFlush(DocumentsWriterPerThread flushingDWPT) throws IOException { + boolean maybeMerge = false; + while (flushingDWPT != null) { + maybeMerge = true; + try { + // flush concurrently without locking + final FlushedSegment newSegment = flushingDWPT.flush(); + finishFlushedSegment(newSegment); + } finally { + flushControl.doAfterFlush(flushingDWPT); + flushingDWPT.checkAndResetHasAborted(); + indexWriter.flushCount.incrementAndGet(); + } + flushingDWPT = flushControl.nextPendingFlush() ; } + return maybeMerge; + } + - if (newSegment != null) { - perThreadPool.clearThreadBindings(perThread); - return true; - } - - return false; - } - - private void finishFlushedSegment(FlushedSegment newSegment) throws IOException { + private void finishFlushedSegment(FlushedSegment newSegment) + throws IOException { pushDeletes(newSegment); if (newSegment != null) { indexWriter.addFlushedSegment(newSegment); - } - } - - private final FlushedSegment finishAddDocument(DocumentsWriterPerThread perThread, - long perThreadRAMUsedBeforeAdd) throws IOException { - FlushedSegment newSegment = null; - final int maxBufferedDocs = config.getMaxBufferedDocs(); - if (maxBufferedDocs != IndexWriterConfig.DISABLE_AUTO_FLUSH && - perThread.getNumDocsInRAM() >= maxBufferedDocs) { - newSegment = perThread.flush(); } - - long deltaRAM = perThread.bytesUsed() - perThreadRAMUsedBeforeAdd; - long oldValue = ramUsed.get(); - while (!ramUsed.compareAndSet(oldValue, oldValue + deltaRAM)) { - oldValue = ramUsed.get(); - } - - return newSegment; } final void subtractFlushedNumDocs(int numFlushed) { @@ -402,66 +437,79 @@ final class DocumentsWriter { final long delGen = bufferedDeletesStream.getNextGen(); // Lock order: DW -> BD if (deletes != null && deletes.any()) { - final FrozenBufferedDeletes packet = new FrozenBufferedDeletes(deletes, delGen); - if (infoStream != null) { - message("flush: push buffered deletes"); - } - bufferedDeletesStream.push(packet); - if (infoStream != null) { - message("flush: delGen=" + packet.gen); - } - } - flushedSegment.segmentInfo.setBufferedDeletesGen(delGen); - } + final FrozenBufferedDeletes packet = new FrozenBufferedDeletes(deletes, + delGen); + if (infoStream != null) { + message("flush: push buffered deletes"); } + bufferedDeletesStream.push(packet); + if (infoStream != null) { + message("flush: delGen=" + packet.gen); + } + } + flushedSegment.segmentInfo.setBufferedDeletesGen(delGen); + } + } private synchronized final void maybePushPendingDeletes() { final long delGen = bufferedDeletesStream.getNextGen(); if (pendingDeletes.any()) { - bufferedDeletesStream.push(new FrozenBufferedDeletes(pendingDeletes, delGen)); + indexWriter.bufferedDeletesStream.push(new FrozenBufferedDeletes( + pendingDeletes, delGen)); pendingDeletes.clear(); - } } + } final boolean flushAllThreads(final boolean flushDeletes) throws IOException { - Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); + final Iterator threadsIterator = perThreadPool.getActivePerThreadsIterator(); boolean anythingFlushed = false; while (threadsIterator.hasNext()) { - FlushedSegment newSegment = null; - - ThreadState perThread = threadsIterator.next(); - perThread.lock(); + final ThreadState perThread = threadsIterator.next(); + final DocumentsWriterPerThread flushingDWPT; + /* + * TODO: maybe we can leverage incoming / indexing threads here if we mark + * all active threads pending so that we don't need to block until we got + * the handle. Yet, we need to figure out how to identify that a certain + * DWPT has been flushed since they are simply replaced once checked out + * for flushing. This would give us another level of concurrency during + * commit. + * + * Maybe we simply iterate them and store the ThreadStates and mark + * all as flushPending and at the same time record the DWPT instance as a + * key for the pending ThreadState. This way we can easily iterate until + * all DWPT have changed. + */ + perThread.lock(); try { - - DocumentsWriterPerThread dwpt = perThread.perThread; - final int numDocs = dwpt.getNumDocsInRAM(); - + if (!perThread.isActive()) { + assert closed; + continue; //this perThread is already done maybe by a concurrently indexing thread + } + final DocumentsWriterPerThread dwpt = perThread.perThread; // Always flush docs if there are any - boolean flushDocs = numDocs > 0; - - String segment = dwpt.getSegment(); - + final boolean flushDocs = dwpt.getNumDocsInRAM() > 0; + final String segment = dwpt.getSegment(); // If we are flushing docs, segment must not be null: assert segment != null || !flushDocs; - if (flushDocs) { - newSegment = dwpt.flush(); - - if (newSegment != null) { - perThreadPool.clearThreadBindings(perThread); - } + // check out and set pending if not already set + flushingDWPT = flushControl.tryCheckoutForFlush(perThread, true); + assert flushingDWPT != null : "DWPT must never be null here since we hold the lock and it holds documents"; + assert dwpt == flushingDWPT : "flushControl returned different DWPT"; + try { + final FlushedSegment newSegment = dwpt.flush(); + anythingFlushed = true; + finishFlushedSegment(newSegment); + } finally { + flushControl.doAfterFlush(flushingDWPT); } + } } finally { perThread.unlock(); } - - if (newSegment != null) { - anythingFlushed = true; - finishFlushedSegment(newSegment); - } } if (!anythingFlushed && flushDeletes) { @@ -471,6 +519,10 @@ final class DocumentsWriter { return anythingFlushed; } + + + + // /* We have three pools of RAM: Postings, byte blocks // * (holds freq/prox posting data) and per-doc buffers diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java new file mode 100644 index 00000000000..24deb76b84a --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java @@ -0,0 +1,263 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.HashMap; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; + +/** + * This class controls {@link DocumentsWriterPerThread} flushing during + * indexing. It tracks the memory consumption per + * {@link DocumentsWriterPerThread} and uses a configured {@link FlushPolicy} to + * decide if a {@link DocumentsWriterPerThread} must flush. + *

+ * In addition to the {@link FlushPolicy} the flush control might set certain + * {@link DocumentsWriterPerThread} as flush pending iff a + * {@link DocumentsWriterPerThread} exceeds the + * {@link IndexWriterConfig#getRAMPerThreadHardLimitMB()} to prevent address + * space exhaustion. + */ +public final class DocumentsWriterFlushControl { + + private final long maxBytesPerDWPT; + private long activeBytes = 0; + private long flushBytes = 0; + private volatile int numPending = 0; + private volatile int numFlushing = 0; + final AtomicBoolean flushDeletes = new AtomicBoolean(false); + + long peakActiveBytes = 0;// only with assert + long peakFlushBytes = 0;// only with assert + long peakNetBytes = 0;// only with assert + private final Healthiness healthiness; + private final DocumentsWriterPerThreadPool perThreadPool; + private final FlushPolicy flushPolicy; + private boolean closed = false; + private final HashMap flushingWriters = new HashMap(); + private final BufferedDeletes pendingDeletes; + + DocumentsWriterFlushControl(FlushPolicy flushPolicy, + DocumentsWriterPerThreadPool threadPool, Healthiness healthiness, + BufferedDeletes pendingDeletes, long maxBytesPerDWPT) { + this.healthiness = healthiness; + this.perThreadPool = threadPool; + this.flushPolicy = flushPolicy; + this.maxBytesPerDWPT = maxBytesPerDWPT; + this.pendingDeletes = pendingDeletes; + } + + public synchronized long activeBytes() { + return activeBytes; + } + + public synchronized long flushBytes() { + return flushBytes; + } + + public synchronized long netBytes() { + return flushBytes + activeBytes; + } + + private void commitPerThreadBytes(ThreadState perThread) { + final long delta = perThread.perThread.bytesUsed() + - perThread.perThreadBytes; + perThread.perThreadBytes += delta; + /* + * We need to differentiate here if we are pending since setFlushPending + * moves the perThread memory to the flushBytes and we could be set to + * pending during a delete + */ + if (perThread.flushPending) { + flushBytes += delta; + } else { + activeBytes += delta; + } + assert updatePeaks(delta); + } + + private boolean updatePeaks(long delta) { + peakActiveBytes = Math.max(peakActiveBytes, activeBytes); + peakFlushBytes = Math.max(peakFlushBytes, flushBytes); + peakNetBytes = Math.max(peakNetBytes, netBytes()); + return true; + } + + synchronized DocumentsWriterPerThread doAfterDocument(ThreadState perThread, + boolean isUpdate) { + commitPerThreadBytes(perThread); + if (!perThread.flushPending) { + if (isUpdate) { + flushPolicy.onUpdate(this, perThread); + } else { + flushPolicy.onInsert(this, perThread); + } + if (!perThread.flushPending && perThread.perThreadBytes > maxBytesPerDWPT) { + // safety check to prevent a single DWPT exceeding its RAM limit. This + // is super + // important since we can not address more than 2048 MB per DWPT + setFlushPending(perThread); + } + } + final DocumentsWriterPerThread flushingDWPT = getFlushIfPending(perThread); + healthiness.updateStalled(this); + return flushingDWPT; + } + + synchronized void doAfterFlush(DocumentsWriterPerThread dwpt) { + assert flushingWriters.containsKey(dwpt); + numFlushing--; + Long bytes = flushingWriters.remove(dwpt); + flushBytes -= bytes.longValue(); + perThreadPool.recycle(dwpt); + healthiness.updateStalled(this); + } + + /** + * Sets flush pending state on the given {@link ThreadState}. The + * {@link ThreadState} must have indexed at least on Document and must not be + * already pending. + */ + public synchronized void setFlushPending(ThreadState perThread) { + assert !perThread.flushPending; + assert perThread.perThread.getNumDocsInRAM() > 0; + perThread.flushPending = true; // write access synced + final long bytes = perThread.perThreadBytes; + flushBytes += bytes; + activeBytes -= bytes; + numPending++; // write access synced + } + + synchronized void doOnAbort(ThreadState state) { + if (state.flushPending) { + flushBytes -= state.perThreadBytes; + } else { + activeBytes -= state.perThreadBytes; + } + // take it out of the loop this DWPT is stale + perThreadPool.replaceForFlush(state, closed); + healthiness.updateStalled(this); + } + + synchronized DocumentsWriterPerThread tryCheckoutForFlush( + ThreadState perThread, boolean setPending) { + if (setPending && !perThread.flushPending) { + setFlushPending(perThread); + } + if (perThread.flushPending) { + // we are pending so all memory is already moved to flushBytes + if (perThread.tryLock()) { + try { + if (perThread.isActive()) { + assert perThread.isHeldByCurrentThread(); + final DocumentsWriterPerThread dwpt; + final long bytes = perThread.perThreadBytes; // do that before + // replace! + dwpt = perThreadPool.replaceForFlush(perThread, closed); + assert !flushingWriters.containsKey(dwpt) : "DWPT is already flushing"; + // record the flushing DWPT to reduce flushBytes in doAfterFlush + flushingWriters.put(dwpt, Long.valueOf(bytes)); + numPending--; // write access synced + numFlushing++; + return dwpt; + } + } finally { + perThread.unlock(); + } + } + } + return null; + } + + DocumentsWriterPerThread getFlushIfPending(ThreadState perThread) { + if (numPending > 0) { + final DocumentsWriterPerThread dwpt = perThread == null ? null + : tryCheckoutForFlush(perThread, false); + if (dwpt == null) { + return nextPendingFlush(); + } + return dwpt; + } + return null; + } + + @Override + public String toString() { + return "DocumentsWriterFlushControl [activeBytes=" + activeBytes + + ", flushBytes=" + flushBytes + "]"; + } + + DocumentsWriterPerThread nextPendingFlush() { + if (numPending > 0) { + final Iterator allActiveThreads = perThreadPool + .getActivePerThreadsIterator(); + while (allActiveThreads.hasNext() && numPending > 0) { + ThreadState next = allActiveThreads.next(); + if (next.flushPending) { + DocumentsWriterPerThread dwpt = tryCheckoutForFlush(next, false); + if (dwpt != null) { + return dwpt; + } + } + } + } + return null; + } + + synchronized void setClosed() { + // set by DW to signal that we should not release new DWPT after close + this.closed = true; + } + + /** + * Returns an iterator that provides access to all currently active {@link ThreadState}s + */ + public Iterator allActiveThreads() { + return perThreadPool.getActivePerThreadsIterator(); + } + + long maxNetBytes() { + return flushPolicy.getMaxNetBytes(); + } + + synchronized void doOnDelete(ThreadState state) { + if (!state.flushPending) { + flushPolicy.onDelete(this, state); + } + } + + /** + * Returns the number of delete terms in the global pool + */ + public int getNumGlobalTermDeletes() { + return pendingDeletes.numTermDeletes.get(); + } + + int numFlushingDWPT() { + return numFlushing; + } + + public void setFlushDeletes() { + flushDeletes.set(true); + } + + int numActiveDWPT() { + return this.perThreadPool.getMaxThreadStates(); + } +} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index d4c1f47a11c..58d1f3aa20d 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -31,7 +31,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BitVector; -import org.apache.lucene.util.ByteBlockPool.DirectAllocator; +import org.apache.lucene.util.ByteBlockPool.Allocator; import org.apache.lucene.util.RamUsageEstimator; public class DocumentsWriterPerThread { @@ -73,17 +73,14 @@ public class DocumentsWriterPerThread { final TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriterPerThread); final TermsHashConsumer freqProxWriter = new FreqProxTermsWriter(); - final InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, - new TermsHash(documentsWriterPerThread, termVectorsWriter, null)); + final InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true, + new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null)); final NormsWriter normsWriter = new NormsWriter(); final DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter); return new DocFieldProcessor(documentsWriterPerThread, docInverter); } }; - // Deletes for our still-in-RAM (to be flushed next) segment - BufferedDeletes pendingDeletes = new BufferedDeletes(false); - static class DocState { final DocumentsWriterPerThread docWriter; Analyzer analyzer; @@ -128,7 +125,7 @@ public class DocumentsWriterPerThread { * currently buffered docs. This resets our state, * discarding any docs added since last flush. */ void abort() throws IOException { - aborting = true; + hasAborted = aborting = true; try { if (infoStream != null) { message("docWriter: now abort"); @@ -152,38 +149,53 @@ public class DocumentsWriterPerThread { final DocumentsWriter parent; final IndexWriter writer; - final Directory directory; final DocState docState; final DocConsumer consumer; + final AtomicLong bytesUsed; + + SegmentWriteState flushState; + //Deletes for our still-in-RAM (to be flushed next) segment + BufferedDeletes pendingDeletes; + String segment; // Current segment we are working on + boolean aborting = false; // True if an abort is pending + boolean hasAborted = false; // True if the last exception throws by #updateDocument was aborting - String segment; // Current segment we are working on - boolean aborting; // True if an abort is pending - + private FieldInfos fieldInfos; private final PrintStream infoStream; private int numDocsInRAM; private int flushedDocCount; - SegmentWriteState flushState; - - final AtomicLong bytesUsed = new AtomicLong(0); - - private FieldInfos fieldInfos; - - public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent, FieldInfos fieldInfos, IndexingChain indexingChain) { + + public DocumentsWriterPerThread(Directory directory, DocumentsWriter parent, + FieldInfos fieldInfos, IndexingChain indexingChain) { this.directory = directory; this.parent = parent; this.fieldInfos = fieldInfos; this.writer = parent.indexWriter; this.infoStream = parent.indexWriter.getInfoStream(); this.docState = new DocState(this); - this.docState.similarityProvider = parent.indexWriter.getConfig().getSimilarityProvider(); + this.docState.similarityProvider = parent.indexWriter.getConfig() + .getSimilarityProvider(); consumer = indexingChain.getChain(this); - } + bytesUsed = new AtomicLong(0); + pendingDeletes = new BufferedDeletes(false); + } + + public DocumentsWriterPerThread(DocumentsWriterPerThread other, FieldInfos fieldInfos) { + this(other.directory, other.parent, fieldInfos, other.parent.chain); + + } void setAborting() { aborting = true; } + + boolean checkAndResetHasAborted() { + final boolean retval = hasAborted; + hasAborted = false; + return retval; + } public void updateDocument(Document doc, Analyzer analyzer, Term delTerm) throws IOException { assert writer.testPoint("DocumentsWriterPerThread addDocument start"); @@ -203,7 +215,7 @@ public class DocumentsWriterPerThread { boolean success = false; try { try { - consumer.processDocument(fieldInfos); + consumer.processDocument(fieldInfos); } finally { docState.clear(); } @@ -251,21 +263,33 @@ public class DocumentsWriterPerThread { void deleteQueries(Query... queries) { if (numDocsInRAM > 0) { - for (Query query : queries) { - pendingDeletes.addQuery(query, numDocsInRAM); + for (Query query : queries) { + pendingDeletes.addQuery(query, numDocsInRAM); + } } } - } void deleteTerms(Term... terms) { if (numDocsInRAM > 0) { - for (Term term : terms) { - pendingDeletes.addTerm(term, numDocsInRAM); + for (Term term : terms) { + pendingDeletes.addTerm(term, numDocsInRAM); + } } } + + /** + * Returns the number of delete terms in this {@link DocumentsWriterPerThread} + */ + public int numDeleteTerms() { + // public for FlushPolicy + return pendingDeletes.numTermDeletes.get(); } - int getNumDocsInRAM() { + /** + * Returns the number of RAM resident documents in this {@link DocumentsWriterPerThread} + */ + public int getNumDocsInRAM() { + // public for FlushPolicy return numDocsInRAM; } @@ -285,7 +309,6 @@ public class DocumentsWriterPerThread { /** Flush all pending docs to a new segment */ FlushedSegment flush() throws IOException { assert numDocsInRAM > 0; - flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos, numDocsInRAM, writer.getConfig().getTermIndexInterval(), fieldInfos.buildSegmentCodecs(true), pendingDeletes); @@ -323,16 +346,17 @@ public class DocumentsWriterPerThread { newSegment.setHasVectors(flushState.hasVectors); if (infoStream != null) { - message("new segment has " + flushState.deletedDocs.count() + " deleted docs"); + message("new segment has " + (flushState.deletedDocs == null ? 0 : flushState.deletedDocs.count()) + " deleted docs"); message("new segment has " + (flushState.hasVectors ? "vectors" : "no vectors")); message("flushedFiles=" + newSegment.files()); message("flushed codecs=" + newSegment.getSegmentCodecs()); } flushedDocCount += flushState.numDocs; - BufferedDeletes segmentDeletes = null; + final BufferedDeletes segmentDeletes; if (pendingDeletes.queries.isEmpty()) { pendingDeletes.clear(); + segmentDeletes = null; } else { segmentDeletes = pendingDeletes; pendingDeletes = new BufferedDeletes(false); @@ -350,7 +374,6 @@ public class DocumentsWriterPerThread { parent.indexWriter.deleter.refresh(segment); } } - abort(); } } @@ -362,7 +385,7 @@ public class DocumentsWriterPerThread { } long bytesUsed() { - return bytesUsed.get(); + return bytesUsed.get() + pendingDeletes.bytesUsed.get(); } FieldInfos getFieldInfos() { @@ -395,11 +418,38 @@ public class DocumentsWriterPerThread { bytesUsed.addAndGet(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT); return b; } + + void recycleIntBlocks(int[][] blocks, int offset, int length) { + bytesUsed.addAndGet(-(length *(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT))); + } - final DirectAllocator byteBlockAllocator = new DirectAllocator(); + final Allocator byteBlockAllocator = new DirectTrackingAllocator(); + + + private class DirectTrackingAllocator extends Allocator { + public DirectTrackingAllocator() { + this(BYTE_BLOCK_SIZE); + } + + public DirectTrackingAllocator(int blockSize) { + super(blockSize); + } + + public byte[] getByteBlock() { + bytesUsed.addAndGet(blockSize); + return new byte[blockSize]; + } + @Override + public void recycleByteBlocks(byte[][] blocks, int start, int end) { + bytesUsed.addAndGet(-((end-start)* blockSize)); + for (int i = start; i < end; i++) { + blocks[i] = null; + } + } + + }; String toMB(long v) { return nf.format(v/1024./1024.); } - } diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java index c8665bb02ea..b3e15112304 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java @@ -9,16 +9,92 @@ import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; import org.apache.lucene.index.codecs.CodecProvider; public abstract class DocumentsWriterPerThreadPool { - final static class ThreadState extends ReentrantLock { - final DocumentsWriterPerThread perThread; + + /** + * {@link ThreadState} references and guards a + * {@link DocumentsWriterPerThread} instance that is used during indexing to + * build a in-memory index segment. {@link ThreadState} also holds all flush + * related per-thread data controlled by {@link DocumentsWriterFlushControl}. + *

+ * A {@link ThreadState}, its methods and members should only accessed by one + * thread a time. Users must acquire the lock via {@link ThreadState#lock()} + * and release the lock in a finally block via {@link ThreadState#unlock()} + * before accessing the state. + */ + @SuppressWarnings("serial") + public final static class ThreadState extends ReentrantLock { + // public for FlushPolicy + DocumentsWriterPerThread perThread; + // write access guarded by DocumentsWriterFlushControl + volatile boolean flushPending = false; + // write access guarded by DocumentsWriterFlushControl + long perThreadBytes = 0; + + // guarded by Reentrant lock + private boolean isActive = true; ThreadState(DocumentsWriterPerThread perThread) { this.perThread = perThread; } + + /** + * Resets the internal {@link DocumentsWriterPerThread} with the given one. + * if the given DWPT is null this ThreadState is marked as inactive and should not be used + * for indexing anymore. + * @see #isActive() + */ + void resetWriter(DocumentsWriterPerThread perThread) { + assert this.isHeldByCurrentThread(); + if (perThread == null) { + isActive = false; + } + this.perThread = perThread; + this.perThreadBytes = 0; + this.flushPending = false; + } + + /** + * Returns true if this ThreadState is still open. This will + * only return false iff the DW has been closed and this + * ThreadState is already checked out for flush. + */ + boolean isActive() { + assert this.isHeldByCurrentThread(); + return isActive; + } + + /** + * Returns the number of currently active bytes in this ThreadState's + * {@link DocumentsWriterPerThread} + */ + public long getBytesUsedPerThread() { + assert this.isHeldByCurrentThread(); + // public for FlushPolicy + return perThreadBytes; + } + + /** + * Returns this {@link ThreadState}s {@link DocumentsWriterPerThread} + */ + public DocumentsWriterPerThread getDocumentsWriterPerThread() { + assert this.isHeldByCurrentThread(); + // public for FlushPolicy + return perThread; + } + + /** + * Returns true iff this {@link ThreadState} is marked as flush + * pending otherwise false + */ + public boolean isFlushPending() { + return flushPending; + } } private final ThreadState[] perThreads; private volatile int numThreadStatesActive; + private CodecProvider codecProvider; + private FieldNumberBiMap globalFieldMap; public DocumentsWriterPerThreadPool(int maxNumPerThreads) { maxNumPerThreads = (maxNumPerThreads < 1) ? IndexWriterConfig.DEFAULT_MAX_THREAD_STATES : maxNumPerThreads; @@ -28,7 +104,8 @@ public abstract class DocumentsWriterPerThreadPool { } public void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) { - final CodecProvider codecProvider = config.getCodecProvider(); + codecProvider = config.getCodecProvider(); + this.globalFieldMap = globalFieldMap; for (int i = 0; i < perThreads.length; i++) { final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecProvider)); perThreads[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain)); @@ -41,26 +118,53 @@ public abstract class DocumentsWriterPerThreadPool { public synchronized ThreadState newThreadState() { if (numThreadStatesActive < perThreads.length) { - ThreadState state = perThreads[numThreadStatesActive]; - numThreadStatesActive++; - return state; + return perThreads[numThreadStatesActive++]; } - return null; } - + + protected DocumentsWriterPerThread replaceForFlush(ThreadState threadState, boolean closed) { + assert threadState.isHeldByCurrentThread(); + final DocumentsWriterPerThread dwpt = threadState.perThread; + if (!closed) { + final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecProvider)); + threadState.resetWriter(new DocumentsWriterPerThread(dwpt, infos)); + } else { + threadState.resetWriter(null); + } + clearThreadBindings(threadState); // TODO - do we need to clear ThreadBindings here since we swap DWPT this is not necessary + return dwpt; + } + + public void recycle(DocumentsWriterPerThread dwpt) { + // don't recycle DWPT by default + } + public abstract ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter, Document doc); public abstract void clearThreadBindings(ThreadState perThread); public abstract void clearAllThreadBindings(); + /** + * Returns an iterator providing access to all {@link ThreadState} + * instances. + */ public Iterator getAllPerThreadsIterator() { return getPerThreadsIterator(this.perThreads.length); } + /** + * Returns an iterator providing access to all active {@link ThreadState} + * instances. + *

+ * Note: The returned iterator will only iterator + * {@link ThreadState}s that are active at the point in time when this method + * has been called. + * + */ public Iterator getActivePerThreadsIterator() { - return getPerThreadsIterator(this.numThreadStatesActive); + return getPerThreadsIterator(numThreadStatesActive); } private Iterator getPerThreadsIterator(final int upto) { @@ -80,4 +184,21 @@ public abstract class DocumentsWriterPerThreadPool { } }; } + + /** + * Returns the ThreadState with the minimum estimated number of threads + * waiting to acquire its lock or null if no {@link ThreadState} + * is yet visible to the calling thread. + */ + protected ThreadState minContendedThreadState() { + ThreadState minThreadState = null; + final Iterator it = getActivePerThreadsIterator(); + while (it.hasNext()) { + final ThreadState state = it.next(); + if (minThreadState == null || state.getQueueLength() < minThreadState.getQueueLength()) { + minThreadState = state; + } + } + return minThreadState; + } } diff --git a/lucene/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java b/lucene/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java new file mode 100644 index 00000000000..1f0b42032f8 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java @@ -0,0 +1,66 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; + +/** + * Default {@link FlushPolicy} implementation that flushes based on RAM + * Consumption, document count and number of buffered deletes depending on the + * IndexWriters {@link IndexWriterConfig}. This {@link FlushPolicy} will only + * respect settings which are not disabled during initialization ( + * {@link #init(DocumentsWriter)}). All enabled {@link IndexWriterConfig} + * settings are used to mark {@link DocumentsWriterPerThread} as flush pending + * during indexing with respect to thier live updates. + *

+ * If {@link IndexWriterConfig#setRAMBufferSizeMB(double)} is enabled always the + * largest ram consuming {@link DocumentsWriterPerThread} will be marked as + * pending iff the global active RAM consumption is equals or higher the + * configured max RAM buffer. + */ +public class FlushByRamOrCountsPolicy extends FlushPolicy { + + @Override + public void onDelete(DocumentsWriterFlushControl control, ThreadState state) { + if (flushOnDeleteTerms()) { + final int maxBufferedDeleteTerms = indexWriterConfig + .getMaxBufferedDeleteTerms(); + if (control.getNumGlobalTermDeletes() >= maxBufferedDeleteTerms) { + control.setFlushDeletes(); + } + } + } + + @Override + public void onInsert(DocumentsWriterFlushControl control, ThreadState state) { + if (flushOnDocCount() + && state.perThread.getNumDocsInRAM() >= indexWriterConfig + .getMaxBufferedDocs()) { + control.setFlushPending(state); // flush by num docs + } else {// flush by RAM + if (flushOnRAM()) { + final double ramBufferSizeMB = indexWriterConfig.getRAMBufferSizeMB(); + final long totalRam = control.activeBytes(); + final long limit = (long) (ramBufferSizeMB * 1024.d * 1024.d); + if (totalRam >= limit) { + markLargestWriterPending(control, state, totalRam); + } + } + } + } +} diff --git a/lucene/src/java/org/apache/lucene/index/FlushPolicy.java b/lucene/src/java/org/apache/lucene/index/FlushPolicy.java new file mode 100644 index 00000000000..e091d3e86d9 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/FlushPolicy.java @@ -0,0 +1,191 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.Iterator; + +import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.SetOnce; + +/** + * {@link FlushPolicy} controls when segments are flushed from a RAM resident + * internal data-structure to the {@link IndexWriter}s {@link Directory}. + *

+ * Segments are traditionally flushed by: + *

    + *
  • RAM consumption - configured via + * {@link IndexWriterConfig#setRAMBufferSizeMB(double)}
    • + *
  • Number of RAM resident documents - configured via + * {@link IndexWriterConfig#setMaxBufferedDocs(int)}
    • + *
  • Number of buffered delete terms - configured via + * {@link IndexWriterConfig#setMaxBufferedDeleteTerms(int)}
    • + *
+ * + * The {@link IndexWriter} uses a provided {@link FlushPolicy} to control the + * flushing process during indexing. The policy is informed for each added or + * updated document as well as for each delete term. Based on the + * {@link FlushPolicy} the information provided via {@link ThreadState} and + * {@link DocumentsWriterFlushControl} the {@link FlushPolicy} can decide if a + * {@link DocumentsWriterPerThread} needs flushing and can mark it as + * flush-pending via + * {@link DocumentsWriterFlushControl#setFlushPending(ThreadState)}. + * + * @see ThreadState + * @see DocumentsWriterFlushControl + * @see DocumentsWriterPerThread + * @see IndexWriterConfig#setFlushPolicy(FlushPolicy) + */ +public abstract class FlushPolicy { + protected final SetOnce writer = new SetOnce(); + protected IndexWriterConfig indexWriterConfig; + + /** + * Called for each delete term applied to the given {@link ThreadState}s + * {@link DocumentsWriterPerThread}. + *

+ * Note: This method is synchronized by the given + * {@link DocumentsWriterFlushControl} and it is guaranteed that the calling + * thread holds the lock on the given {@link ThreadState} + */ + public abstract void onDelete(DocumentsWriterFlushControl control, + ThreadState state); + + /** + * Called for each document update on the given {@link ThreadState}s + * {@link DocumentsWriterPerThread}. + *

+ * Note: This method is synchronized by the given + * {@link DocumentsWriterFlushControl} and it is guaranteed that the calling + * thread holds the lock on the given {@link ThreadState} + */ + public void onUpdate(DocumentsWriterFlushControl control, ThreadState state) { + onInsert(control, state); + if (!state.flushPending) { + onDelete(control, state); + } + } + + /** + * Called for each document addition on the given {@link ThreadState}s + * {@link DocumentsWriterPerThread}. + *

+ * Note: This method is synchronized by the given + * {@link DocumentsWriterFlushControl} and it is guaranteed that the calling + * thread holds the lock on the given {@link ThreadState} + */ + public abstract void onInsert(DocumentsWriterFlushControl control, + ThreadState state); + + /** + * Called by {@link DocumentsWriter} to initialize the FlushPolicy + */ + protected synchronized void init(DocumentsWriter docsWriter) { + writer.set(docsWriter); + indexWriterConfig = docsWriter.indexWriter.getConfig(); + } + + /** + * Marks the most ram consuming active {@link DocumentsWriterPerThread} flush + * pending + */ + protected void markLargestWriterPending(DocumentsWriterFlushControl control, + ThreadState perThreadState, final long currentBytesPerThread) { + control + .setFlushPending(findLargestNonPendingWriter(control, perThreadState)); + } + + /** + * Returns the current most RAM consuming non-pending {@link ThreadState} with + * at least one indexed document. + *

+ * This method will never return null + */ + protected ThreadState findLargestNonPendingWriter( + DocumentsWriterFlushControl control, ThreadState perThreadState) { + long maxRamSoFar = perThreadState.perThreadBytes; + // the dwpt which needs to be flushed eventually + ThreadState maxRamUsingThreadState = perThreadState; + assert !perThreadState.flushPending : "DWPT should have flushed"; + Iterator activePerThreadsIterator = control.allActiveThreads(); + while (activePerThreadsIterator.hasNext()) { + ThreadState next = activePerThreadsIterator.next(); + if (!next.flushPending) { + final long nextRam = next.perThreadBytes; + if (nextRam > maxRamSoFar && next.perThread.getNumDocsInRAM() > 0) { + maxRamSoFar = nextRam; + maxRamUsingThreadState = next; + } + } + } + assert maxRamUsingThreadState.perThread.getNumDocsInRAM() > 0; + assert writer.get().message( + "set largest ram consuming thread pending on lower watermark"); + return maxRamUsingThreadState; + } + + /** + * Returns the max net memory which marks the upper watermark for the + * DocumentsWriter to be healthy. If all flushing and active + * {@link DocumentsWriterPerThread} consume more memory than the upper + * watermark all incoming threads should be stalled and blocked until the + * memory drops below this. + *

+ * Note: the upper watermark is only taken into account if this + * {@link FlushPolicy} flushes by ram usage. + * + *

+ * The default for the max net memory is set to 2 x + * {@link IndexWriterConfig#getRAMBufferSizeMB()} + * + */ + public long getMaxNetBytes() { + if (!flushOnRAM()) { + return -1; + } + final double ramBufferSizeMB = indexWriterConfig.getRAMBufferSizeMB(); + return (long) (ramBufferSizeMB * 1024.d * 1024.d * 2); + } + + /** + * Returns true if this {@link FlushPolicy} flushes on + * {@link IndexWriterConfig#getMaxBufferedDocs()}, otherwise + * false. + */ + protected boolean flushOnDocCount() { + return indexWriterConfig.getMaxBufferedDocs() != IndexWriterConfig.DISABLE_AUTO_FLUSH; + } + + /** + * Returns true if this {@link FlushPolicy} flushes on + * {@link IndexWriterConfig#getMaxBufferedDeleteTerms()}, otherwise + * false. + */ + protected boolean flushOnDeleteTerms() { + return indexWriterConfig.getMaxBufferedDeleteTerms() != IndexWriterConfig.DISABLE_AUTO_FLUSH; + } + + /** + * Returns true if this {@link FlushPolicy} flushes on + * {@link IndexWriterConfig#getRAMBufferSizeMB()}, otherwise + * false. + */ + protected boolean flushOnRAM() { + return indexWriterConfig.getRAMBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH; + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/Healthiness.java b/lucene/src/java/org/apache/lucene/index/Healthiness.java new file mode 100644 index 00000000000..716e0ff9a05 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/Healthiness.java @@ -0,0 +1,120 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.concurrent.locks.AbstractQueuedSynchronizer; + +import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; + +/** + * Controls the health status of a {@link DocumentsWriter} sessions. This class + * used to block incoming indexing threads if flushing significantly slower than + * indexing to ensure the {@link DocumentsWriter}s healthiness. If flushing is + * significantly slower than indexing the net memory used within an + * {@link IndexWriter} session can increase very quickly and easily exceed the + * JVM's available memory. + *

+ * To prevent OOM Errors and ensure IndexWriter's stability this class blocks + * incoming threads from indexing once 2 x number of available + * {@link ThreadState}s in {@link DocumentsWriterPerThreadPool} is exceeded. + * Once flushing catches up and the number of flushing DWPT is equal or lower + * than the number of active {@link ThreadState}s threads are released and can + * continue indexing. + */ +final class Healthiness { + + @SuppressWarnings("serial") + private static final class Sync extends AbstractQueuedSynchronizer { + volatile boolean hasBlockedThreads = false; // only with assert + + Sync() { + setState(0); + } + + boolean isHealthy() { + return getState() == 0; + } + + boolean trySetStalled() { + int state = getState(); + return compareAndSetState(state, state + 1); + } + + boolean tryReset() { + final int oldState = getState(); + if (oldState == 0) + return true; + if (compareAndSetState(oldState, 0)) { + releaseShared(0); + return true; + } + return false; + } + + @Override + public int tryAcquireShared(int acquires) { + assert maybeSetHasBlocked(getState()); + return getState() == 0 ? 1 : -1; + } + + // only used for testing + private boolean maybeSetHasBlocked(int state) { + hasBlockedThreads |= getState() != 0; + return true; + } + + @Override + public boolean tryReleaseShared(int newState) { + return (getState() == 0); + } + } + + private final Healthiness.Sync sync = new Sync(); + volatile boolean wasStalled = false; // only with asserts + + boolean isStalled() { + return !sync.isHealthy(); + } + + /** + * Update the stalled flag status. This method will set the stalled flag to + * true iff the number of flushing + * {@link DocumentsWriterPerThread} is greater than the number of active + * {@link DocumentsWriterPerThread}. Otherwise it will reset the + * {@link Healthiness} to healthy and release all threads waiting on + * {@link #waitIfStalled()} + */ + void updateStalled(DocumentsWriterFlushControl flushControl) { + do { + // if we have more flushing DWPT than numActiveDWPT we stall! + while (flushControl.numActiveDWPT() < flushControl.numFlushingDWPT()) { + if (sync.trySetStalled()) { + assert wasStalled = true; + return; + } + } + } while (!sync.tryReset()); + } + + void waitIfStalled() { + sync.acquireShared(0); + } + + boolean hasBlocked() { + return sync.hasBlockedThreads; + } +} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 7c83eb19c65..622b5068dfd 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -248,8 +248,8 @@ public class IndexWriter implements Closeable { private long mergeGen; private boolean stopMerges; - private final AtomicInteger flushCount = new AtomicInteger(); - private final AtomicInteger flushDeletesCount = new AtomicInteger(); + final AtomicInteger flushCount = new AtomicInteger(); + final AtomicInteger flushDeletesCount = new AtomicInteger(); final ReaderPool readerPool = new ReaderPool(); final BufferedDeletesStream bufferedDeletesStream; @@ -2540,17 +2540,7 @@ public class IndexWriter implements Closeable { doBeforeFlush(); assert testPoint("startDoFlush"); - - // We may be flushing because it was triggered by doc - // count, del count, ram usage (in which case flush - // pending is already set), or we may be flushing - // due to external event eg getReader or commit is - // called (in which case we now set it, and this will - // pause all threads): - flushControl.setFlushPendingNoWait("explicit flush"); - boolean success = false; - try { if (infoStream != null) { @@ -2566,8 +2556,7 @@ public class IndexWriter implements Closeable { // buffer, force them all to apply now. This is to // prevent too-frequent flushing of a long tail of // tiny segments: - if (flushControl.getFlushDeletes() || - (config.getRAMBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH && + if ((config.getRAMBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH && bufferedDeletesStream.bytesUsed() > (1024*1024*config.getRAMBufferSizeMB()/2))) { applyAllDeletes = true; if (infoStream != null) { @@ -2580,39 +2569,16 @@ public class IndexWriter implements Closeable { if (infoStream != null) { message("apply all deletes during flush"); } - flushDeletesCount.incrementAndGet(); - final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, segmentInfos); - if (result.anyDeletes) { - checkpoint(); - } - if (!keepFullyDeletedSegments && result.allDeleted != null) { - if (infoStream != null) { - message("drop 100% deleted segments: " + result.allDeleted); - } - for(SegmentInfo info : result.allDeleted) { - // If a merge has already registered for this - // segment, we leave it in the readerPool; the - // merge will skip merging it and will then drop - // it once it's done: - if (!mergingSegments.contains(info)) { - segmentInfos.remove(info); - if (readerPool != null) { - readerPool.drop(info); - } - } - } - checkpoint(); - } - bufferedDeletesStream.prune(segmentInfos); - assert !bufferedDeletesStream.any(); - - flushControl.clearDeletes(); + applyAllDeletes(); } else if (infoStream != null) { message("don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed()); } doAfterFlush(); - flushCount.incrementAndGet(); + if (!maybeMerge) { + // flushCount is incremented in flushAllThreads + flushCount.incrementAndGet(); + } success = true; @@ -2624,20 +2590,51 @@ public class IndexWriter implements Closeable { // never hit return false; } finally { - flushControl.clearFlushPending(); if (!success && infoStream != null) message("hit exception during flush"); } } + + final synchronized void applyAllDeletes() throws IOException { + flushDeletesCount.incrementAndGet(); + final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, segmentInfos); + if (result.anyDeletes) { + checkpoint(); + } + if (!keepFullyDeletedSegments && result.allDeleted != null) { + if (infoStream != null) { + message("drop 100% deleted segments: " + result.allDeleted); + } + for(SegmentInfo info : result.allDeleted) { + // If a merge has already registered for this + // segment, we leave it in the readerPool; the + // merge will skip merging it and will then drop + // it once it's done: + if (!mergingSegments.contains(info)) { + segmentInfos.remove(info); + if (readerPool != null) { + readerPool.drop(info); + } + } + } + checkpoint(); + } + bufferedDeletesStream.prune(segmentInfos); + } /** Expert: Return the total size of all index files currently cached in memory. * Useful for size management with flushRamDocs() */ public final long ramSizeInBytes() { ensureOpen(); - // nocommit - //return docWriter.bytesUsed() + bufferedDeletesStream.bytesUsed(); - return 0; + return docWriter.flushControl.netBytes() + bufferedDeletesStream.bytesUsed(); + } + + // for testing only + DocumentsWriter getDocsWriter() { + boolean test = false; + assert test = true; + return test?docWriter: null; } /** Expert: Return the number of documents currently @@ -3681,124 +3678,4 @@ public class IndexWriter implements Closeable { public PayloadProcessorProvider getPayloadProcessorProvider() { return payloadProcessorProvider; } - - // decides when flushes happen - final class FlushControl { - - private boolean flushPending; - private boolean flushDeletes; - private int delCount; - private int docCount; - private boolean flushing; - - private synchronized boolean setFlushPending(String reason, boolean doWait) { - if (flushPending || flushing) { - if (doWait) { - while(flushPending || flushing) { - try { - wait(); - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - } - } - return false; - } else { - if (infoStream != null) { - message("now trigger flush reason=" + reason); - } - flushPending = true; - return flushPending; - } - } - - public synchronized void setFlushPendingNoWait(String reason) { - setFlushPending(reason, false); - } - - public synchronized boolean getFlushPending() { - return flushPending; - } - - public synchronized boolean getFlushDeletes() { - return flushDeletes; - } - - public synchronized void clearFlushPending() { - if (infoStream != null) { - message("clearFlushPending"); - } - flushPending = false; - flushDeletes = false; - docCount = 0; - notifyAll(); - } - - public synchronized void clearDeletes() { - delCount = 0; - } - - public synchronized boolean waitUpdate(int docInc, int delInc) { - return waitUpdate(docInc, delInc, false); - } - - public synchronized boolean waitUpdate(int docInc, int delInc, boolean skipWait) { - while(flushPending) { - try { - wait(); - } catch (InterruptedException ie) { - throw new ThreadInterruptedException(ie); - } - } - - // skipWait is only used when a thread is BOTH adding - // a doc and buffering a del term, and, the adding of - // the doc already triggered a flush - if (skipWait) { - docCount += docInc; - delCount += delInc; - return false; - } - - final int maxBufferedDocs = config.getMaxBufferedDocs(); - if (maxBufferedDocs != IndexWriterConfig.DISABLE_AUTO_FLUSH && - (docCount+docInc) >= maxBufferedDocs) { - return setFlushPending("maxBufferedDocs", true); - } - docCount += docInc; - - final int maxBufferedDeleteTerms = config.getMaxBufferedDeleteTerms(); - if (maxBufferedDeleteTerms != IndexWriterConfig.DISABLE_AUTO_FLUSH && - (delCount+delInc) >= maxBufferedDeleteTerms) { - flushDeletes = true; - return setFlushPending("maxBufferedDeleteTerms", true); - } - delCount += delInc; - - return flushByRAMUsage("add delete/doc"); - } - - public synchronized boolean flushByRAMUsage(String reason) { -// final double ramBufferSizeMB = config.getRAMBufferSizeMB(); -// if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH) { -// final long limit = (long) (ramBufferSizeMB*1024*1024); -// long used = bufferedDeletesStream.bytesUsed() + docWriter.bytesUsed(); -// if (used >= limit) { -// -// // DocumentsWriter may be able to free up some -// // RAM: -// // Lock order: FC -> DW -// docWriter.balanceRAM(); -// -// used = bufferedDeletesStream.bytesUsed() + docWriter.bytesUsed(); -// if (used >= limit) { -// return setFlushPending("ram full: " + reason, false); -// } -// } -// } - return false; - } - } - - final FlushControl flushControl = new FlushControl(); } diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java index e6ce95dd076..d6067174e4d 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -94,6 +94,8 @@ public final class IndexWriterConfig implements Cloneable { /** Default value is 1. Change using {@link #setReaderTermsIndexDivisor(int)}. */ public static final int DEFAULT_READER_TERMS_INDEX_DIVISOR = IndexReader.DEFAULT_TERMS_INDEX_DIVISOR; + /** Default value is 1945. Change using {@link #setRAMPerThreadHardLimitMB(int)} */ + public static final int DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB = 1945; /** * Sets the default (for any instance) maximum time to wait for a write lock * (in milliseconds). @@ -130,6 +132,8 @@ public final class IndexWriterConfig implements Cloneable { private volatile DocumentsWriterPerThreadPool indexerThreadPool; private volatile boolean readerPooling; private volatile int readerTermsIndexDivisor; + private volatile FlushPolicy flushPolicy; + private volatile int perThreadHardLimitMB; private Version matchVersion; @@ -160,6 +164,7 @@ public final class IndexWriterConfig implements Cloneable { readerPooling = DEFAULT_READER_POOLING; indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(DEFAULT_MAX_THREAD_STATES); readerTermsIndexDivisor = DEFAULT_READER_TERMS_INDEX_DIVISOR; + perThreadHardLimitMB = DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB; } @Override @@ -352,6 +357,7 @@ public final class IndexWriterConfig implements Cloneable { * @throws IllegalArgumentException if maxBufferedDeleteTerms * is enabled but smaller than 1 * @see #setRAMBufferSizeMB + * @see #setFlushPolicy(FlushPolicy) * *

Takes effect immediately, but only the next time a * document is added, updated or deleted. @@ -380,14 +386,20 @@ public final class IndexWriterConfig implements Cloneable { * and deletions before they are flushed to the Directory. Generally for * faster indexing performance it's best to flush by RAM usage instead of * document count and use as large a RAM buffer as you can. - * *

* When this is set, the writer will flush whenever buffered documents and * deletions use this much RAM. Pass in {@link #DISABLE_AUTO_FLUSH} to prevent * triggering a flush due to RAM usage. Note that if flushing by document * count is also enabled, then the flush will be triggered by whichever comes * first. - * + *

+ * The maximum RAM limit is inherently determined by the JVMs available memory. + * Yet, an {@link IndexWriter} session can consume a significantly larger amount + * of memory than the given RAM limit since this limit is just an indicator when + * to flush memory resident documents to the Directory. Flushes are likely happen + * concurrently while other threads adding documents to the writer. For application + * stability the available memory in the JVM should be significantly larger than + * the RAM buffer used for indexing. *

* NOTE: the account of RAM usage for pending deletions is only * approximate. Specifically, if you delete by Query, Lucene currently has no @@ -396,16 +408,15 @@ public final class IndexWriterConfig implements Cloneable { * periodically yourself, or by using {@link #setMaxBufferedDeleteTerms(int)} * to flush by count instead of RAM usage (each buffered delete Query counts * as one). - * *

- * NOTE: because IndexWriter uses ints when managing its - * internal storage, the absolute maximum value for this setting is somewhat - * less than 2048 MB. The precise limit depends on various factors, such as - * how large your documents are, how many fields have norms, etc., so it's - * best to set this value comfortably under 2048. - * + * NOTE: It's not guaranteed that all memory resident documents are flushed + * once this limit is exceeded. Depending on the configured {@link FlushPolicy} only a + * subset of the buffered documents are flushed and therefore only parts of the RAM + * buffer is released. *

+ * * The default value is {@link #DEFAULT_RAM_BUFFER_SIZE_MB}. + * @see #setFlushPolicy(FlushPolicy) * *

Takes effect immediately, but only the next time a * document is added, updated or deleted. @@ -413,12 +424,9 @@ public final class IndexWriterConfig implements Cloneable { * @throws IllegalArgumentException * if ramBufferSize is enabled but non-positive, or it disables * ramBufferSize when maxBufferedDocs is already disabled + * */ public IndexWriterConfig setRAMBufferSizeMB(double ramBufferSizeMB) { - if (ramBufferSizeMB > 2048.0) { - throw new IllegalArgumentException("ramBufferSize " + ramBufferSizeMB - + " is too large; should be comfortably less than 2048"); - } if (ramBufferSizeMB != DISABLE_AUTO_FLUSH && ramBufferSizeMB <= 0.0) throw new IllegalArgumentException( "ramBufferSize should be > 0.0 MB when enabled"); @@ -453,7 +461,7 @@ public final class IndexWriterConfig implements Cloneable { * document is added, updated or deleted. * * @see #setRAMBufferSizeMB(double) - * + * @see #setFlushPolicy(FlushPolicy) * @throws IllegalArgumentException * if maxBufferedDocs is enabled but smaller than 2, or it disables * maxBufferedDocs when ramBufferSize is already disabled @@ -607,6 +615,53 @@ public final class IndexWriterConfig implements Cloneable { public int getReaderTermsIndexDivisor() { return readerTermsIndexDivisor; } + + /** + * Expert: Controls when segments are flushed to disk during indexing. + * The {@link FlushPolicy} initialized during {@link IndexWriter} instantiation and once initialized + * the given instance is bound to this {@link IndexWriter} and should not be used with another writer. + * @see #setMaxBufferedDeleteTerms(int) + * @see #setMaxBufferedDocs(int) + * @see #setRAMBufferSizeMB(double) + */ + public IndexWriterConfig setFlushPolicy(FlushPolicy flushPolicy) { + this.flushPolicy = flushPolicy; + return this; + } + + /** + * Expert: Sets the maximum memory consumption per thread triggering a forced + * flush if exceeded. A {@link DocumentsWriterPerThread} is forcefully flushed + * once it exceeds this limit even if the {@link #getRAMBufferSizeMB()} has + * not been exceeded. This is a safety limit to prevent a + * {@link DocumentsWriterPerThread} from address space exhaustion due to its + * internal 32 bit signed integer based memory addressing. + * The given value must be less that 2GB (2048MB) + * + * @see #DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB + */ + public IndexWriterConfig setRAMPerThreadHardLimitMB(int perThreadHardLimitMB) { + if (perThreadHardLimitMB <= 0 || perThreadHardLimitMB >= 2048) { + throw new IllegalArgumentException("PerThreadHardLimit must be greater than 0 and less than 2048MB"); + } + this.perThreadHardLimitMB = perThreadHardLimitMB; + return this; + } + + /** + * Returns the max amount of memory each {@link DocumentsWriterPerThread} can + * consume until forcefully flushed. + * @see #setRAMPerThreadHardLimitMB(int) + */ + public int getRAMPerThreadHardLimitMB() { + return perThreadHardLimitMB; + } + /** + * @see #setFlushPolicy(FlushPolicy) + */ + public FlushPolicy getFlushPolicy() { + return flushPolicy; + } @Override public String toString() { @@ -631,6 +686,10 @@ public final class IndexWriterConfig implements Cloneable { sb.append("maxThreadStates=").append(indexerThreadPool.getMaxThreadStates()).append("\n"); sb.append("readerPooling=").append(readerPooling).append("\n"); sb.append("readerTermsIndexDivisor=").append(readerTermsIndexDivisor).append("\n"); + sb.append("flushPolicy=").append(flushPolicy).append("\n"); + sb.append("perThreadHardLimitMB=").append(perThreadHardLimitMB).append("\n"); + return sb.toString(); } + } diff --git a/lucene/src/java/org/apache/lucene/index/IntBlockPool.java b/lucene/src/java/org/apache/lucene/index/IntBlockPool.java index eb85f0ad9c1..16093a5c34e 100644 --- a/lucene/src/java/org/apache/lucene/index/IntBlockPool.java +++ b/lucene/src/java/org/apache/lucene/index/IntBlockPool.java @@ -1,5 +1,7 @@ package org.apache.lucene.index; +import java.util.Arrays; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -36,6 +38,10 @@ final class IntBlockPool { public void reset() { if (bufferUpto != -1) { // Reuse first buffer + if (bufferUpto > 0) { + docWriter.recycleIntBlocks(buffers, 1, bufferUpto-1); + Arrays.fill(buffers, 1, bufferUpto, null); + } bufferUpto = 0; intUpto = 0; intOffset = 0; diff --git a/lucene/src/java/org/apache/lucene/index/TermsHash.java b/lucene/src/java/org/apache/lucene/index/TermsHash.java index ede19f7c4dc..af51996d0fa 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsHash.java +++ b/lucene/src/java/org/apache/lucene/index/TermsHash.java @@ -52,13 +52,14 @@ final class TermsHash extends InvertedDocConsumer { // Used by perField to obtain terms from the analysis chain final BytesRef termBytesRef = new BytesRef(10); - boolean trackAllocations; + final boolean trackAllocations; - public TermsHash(final DocumentsWriterPerThread docWriter, final TermsHashConsumer consumer, final TermsHash nextTermsHash) { + public TermsHash(final DocumentsWriterPerThread docWriter, final TermsHashConsumer consumer, boolean trackAllocations, final TermsHash nextTermsHash) { this.docState = docWriter.docState; this.docWriter = docWriter; this.consumer = consumer; + this.trackAllocations = trackAllocations; this.nextTermsHash = nextTermsHash; intPool = new IntBlockPool(docWriter); bytePool = new ByteBlockPool(docWriter.byteBlockAllocator); diff --git a/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java b/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java index ca26a8f2f26..3f735ba7434 100644 --- a/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java +++ b/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java @@ -63,8 +63,8 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { termBytePool = termsHash.termBytePool; docState = termsHash.docState; this.termsHash = termsHash; - bytesUsed = termsHash.trackAllocations?termsHash.docWriter.bytesUsed:new AtomicLong(); - + bytesUsed = termsHash.trackAllocations ? termsHash.docWriter.bytesUsed + : new AtomicLong(); fieldState = docInverterPerField.fieldState; this.consumer = termsHash.consumer.addField(this, fieldInfo); PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, bytesUsed); @@ -311,7 +311,7 @@ final class TermsHashPerField extends InvertedDocConsumerPerField { @Override public int[] clear() { if(perField.postingsArray != null) { - bytesUsed.addAndGet(-perField.postingsArray.size * perField.postingsArray.bytesPerPosting()); + bytesUsed.addAndGet(-(perField.postingsArray.size * perField.postingsArray.bytesPerPosting())); perField.postingsArray = null; } return null; diff --git a/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java b/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java index cba0fc98beb..b00e85a97a9 100644 --- a/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java @@ -1,6 +1,20 @@ package org.apache.lucene.index; - -import java.util.Iterator; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -11,6 +25,7 @@ public class ThreadAffinityDocumentsWriterThreadPool extends DocumentsWriterPerT public ThreadAffinityDocumentsWriterThreadPool(int maxNumPerThreads) { super(maxNumPerThreads); + assert getMaxThreadStates() >= 1; } @Override @@ -21,25 +36,25 @@ public class ThreadAffinityDocumentsWriterThreadPool extends DocumentsWriterPerT return threadState; } } - - // find the state that has minimum amount of threads waiting - Iterator it = getActivePerThreadsIterator(); ThreadState minThreadState = null; - while (it.hasNext()) { - ThreadState state = it.next(); - if (minThreadState == null || state.getQueueLength() < minThreadState.getQueueLength()) { - minThreadState = state; - } - } - + // find the state that has minimum amount of threads waiting + minThreadState = minContendedThreadState(); if (minThreadState == null || minThreadState.hasQueuedThreads()) { ThreadState newState = newThreadState(); if (newState != null) { minThreadState = newState; threadBindings.put(requestingThread, newState); + } else if (minThreadState == null) { + /* + * no new threadState available we just take the minContented one + * This must return a valid thread state since we accessed the + * synced context in newThreadState() above. + */ + minThreadState = minContendedThreadState(); } } - + assert minThreadState != null: "ThreadState is null"; + minThreadState.lock(); return minThreadState; } @@ -53,4 +68,5 @@ public class ThreadAffinityDocumentsWriterThreadPool extends DocumentsWriterPerT public void clearAllThreadBindings() { threadBindings.clear(); } + } diff --git a/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java index bb9552bf7d9..75ee555d193 100644 --- a/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java @@ -33,6 +33,7 @@ import java.util.Set; import org.apache.lucene.index.IndexReader; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.ThrottledIndexOutput; import org.apache.lucene.util._TestUtil; /** @@ -56,6 +57,7 @@ public class MockDirectoryWrapper extends Directory { private Set createdFiles; Set openFilesForWrite = new HashSet(); volatile boolean crashed; + private ThrottledIndexOutput throttledOutput; // use this for tracking files for crash. // additionally: provides debugging information in case you leave one open @@ -101,6 +103,10 @@ public class MockDirectoryWrapper extends Directory { public void setPreventDoubleWrite(boolean value) { preventDoubleWrite = value; } + + public void setThrottledIndexOutput(ThrottledIndexOutput throttledOutput) { + this.throttledOutput = throttledOutput; + } @Override public synchronized void sync(Collection names) throws IOException { @@ -335,7 +341,7 @@ public class MockDirectoryWrapper extends Directory { IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name); openFileHandles.put(io, new RuntimeException("unclosed IndexOutput")); openFilesForWrite.add(name); - return io; + return throttledOutput == null ? io : throttledOutput.newFromDelegate(io); } @Override @@ -547,4 +553,5 @@ public class MockDirectoryWrapper extends Directory { maybeYield(); delegate.copy(to, src, dest); } + } diff --git a/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java b/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java new file mode 100644 index 00000000000..52333bd2cd7 --- /dev/null +++ b/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java @@ -0,0 +1,147 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.IndexOutput; + +public class ThrottledIndexOutput extends IndexOutput { + public static final int DEFAULT_MIN_WRITTEN_BYTES = 1024; + private final int bytesPerSecond; + private IndexOutput delegate; + private long flushDelayMillis; + private long closeDelayMillis; + private long seekDelayMillis; + private long pendingBytes; + private long minBytesWritten; + private long timeElapsed; + private final byte[] bytes = new byte[1]; + + public ThrottledIndexOutput newFromDelegate(IndexOutput output) { + return new ThrottledIndexOutput(bytesPerSecond, flushDelayMillis, + closeDelayMillis, seekDelayMillis, minBytesWritten, output); + } + + public ThrottledIndexOutput(int bytesPerSecond, long delayInMillis, + IndexOutput delegate) { + this(bytesPerSecond, delayInMillis, delayInMillis, delayInMillis, + DEFAULT_MIN_WRITTEN_BYTES, delegate); + } + + public ThrottledIndexOutput(int bytesPerSecond, long delays, + int minBytesWritten, IndexOutput delegate) { + this(bytesPerSecond, delays, delays, delays, minBytesWritten, delegate); + } + + public static final int mBitsToBytes(int mbits) { + return mbits * 125000; + } + + public ThrottledIndexOutput(int bytesPerSecond, long flushDelayMillis, + long closeDelayMillis, long seekDelayMillis, long minBytesWritten, + IndexOutput delegate) { + assert bytesPerSecond > 0; + this.delegate = delegate; + this.bytesPerSecond = bytesPerSecond; + this.flushDelayMillis = flushDelayMillis; + this.closeDelayMillis = closeDelayMillis; + this.seekDelayMillis = seekDelayMillis; + this.minBytesWritten = minBytesWritten; + } + + @Override + public void flush() throws IOException { + sleep(flushDelayMillis); + delegate.flush(); + } + + @Override + public void close() throws IOException { + sleep(closeDelayMillis + getDelay(true)); + delegate.close(); + + } + + @Override + public long getFilePointer() { + return delegate.getFilePointer(); + } + + @Override + public void seek(long pos) throws IOException { + sleep(seekDelayMillis); + delegate.seek(pos); + } + + @Override + public long length() throws IOException { + return delegate.length(); + } + + @Override + public void writeByte(byte b) throws IOException { + bytes[0] = b; + writeBytes(bytes, 0, 1); + } + + @Override + public void writeBytes(byte[] b, int offset, int length) throws IOException { + final long before = System.nanoTime(); + delegate.writeBytes(b, offset, length); + timeElapsed += System.nanoTime() - before; + pendingBytes += length; + sleep(getDelay(false)); + + } + + protected long getDelay(boolean closing) { + if (pendingBytes > 0 && (closing || pendingBytes > minBytesWritten)) { + long actualBps = (timeElapsed / pendingBytes) * 1000000000l; // nano to sec + if (actualBps > bytesPerSecond) { + long expected = (pendingBytes * 1000l / bytesPerSecond) ; + final long delay = expected - (timeElapsed / 1000000l) ; + pendingBytes = 0; + timeElapsed = 0; + return delay; + } + } + return 0; + + } + + private static final void sleep(long ms) { + if (ms <= 0) + return; + try { + Thread.sleep(ms); + } catch (InterruptedException e) { + throw new ThreadInterruptedException(e); + } + } + + @Override + public void setLength(long length) throws IOException { + delegate.setLength(length); + } + + @Override + public void copyBytes(DataInput input, long numBytes) throws IOException { + delegate.copyBytes(input, numBytes); + } +} diff --git a/lucene/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java b/lucene/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java new file mode 100644 index 00000000000..9d4286ea5ee --- /dev/null +++ b/lucene/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java @@ -0,0 +1,458 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.ThrottledIndexOutput; +import org.junit.Before; + +public class TestFlushByRamOrCountsPolicy extends LuceneTestCase { + + private LineFileDocs lineDocFile; + private int numCPUs; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + lineDocFile = new LineFileDocs(random); + numCPUs = Runtime.getRuntime().availableProcessors(); + } + + public void testFlushByRam() throws CorruptIndexException, + LockObtainFailedException, IOException, InterruptedException { + int[] numThreads = new int[] { numCPUs + random.nextInt(numCPUs + 1), 1 }; + for (int i = 0; i < numThreads.length; i++) { + runFlushByRam(numThreads[i], + 1 + random.nextInt(10) + random.nextDouble(), false); + } + + for (int i = 0; i < numThreads.length; i++) { + // with a 250 mb ram buffer we should never stall + runFlushByRam(numThreads[i], 250.d, true); + } + } + + protected void runFlushByRam(int numThreads, double maxRam, + boolean ensureNotStalled) throws IOException, CorruptIndexException, + LockObtainFailedException, InterruptedException { + final int numDocumentsToIndex = 50 + random.nextInt(150); + AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); + Directory dir = newDirectory(); + MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()).setFlushPolicy(flushPolicy); + + final int numDWPT = 1 + random.nextInt(8); + DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool( + numDWPT); + iwc.setIndexerThreadPool(threadPool); + iwc.setRAMBufferSizeMB(1 + random.nextInt(10) + random.nextDouble()); + iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); + IndexWriter writer = new IndexWriter(dir, iwc); + assertFalse(flushPolicy.flushOnDocCount()); + assertFalse(flushPolicy.flushOnDeleteTerms()); + assertTrue(flushPolicy.flushOnRAM()); + DocumentsWriter docsWriter = writer.getDocsWriter(); + assertNotNull(docsWriter); + DocumentsWriterFlushControl flushControl = docsWriter.flushControl; + assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes()); + + IndexThread[] threads = new IndexThread[numThreads]; + for (int x = 0; x < threads.length; x++) { + threads[x] = new IndexThread(numDocs, numThreads, writer, lineDocFile, + false); + threads[x].start(); + } + + for (int x = 0; x < threads.length; x++) { + threads[x].join(); + } + final long maxRAMBytes = (long) (iwc.getRAMBufferSizeMB() * 1024. * 1024.); + assertEquals(" all flushes must be due numThreads=" + numThreads, 0, + flushControl.flushBytes()); + assertEquals(numDocumentsToIndex, writer.numDocs()); + assertEquals(numDocumentsToIndex, writer.maxDoc()); + assertTrue("peak bytes without flush exceeded watermark", + flushPolicy.peakBytesWithoutFlush <= maxRAMBytes); + assertActiveBytesAfter(flushControl); + if (flushPolicy.hasMarkedPending) { + assertTrue(maxRAMBytes < flushControl.peakActiveBytes); + } + if (ensureNotStalled) { + assertFalse(docsWriter.healthiness.wasStalled); + } + writer.close(); + assertEquals(0, flushControl.activeBytes()); + dir.close(); + } + + public void testFlushDocCount() throws CorruptIndexException, + LockObtainFailedException, IOException, InterruptedException { + int[] numThreads = new int[] { numCPUs + random.nextInt(numCPUs + 1), 1 }; + for (int i = 0; i < numThreads.length; i++) { + + final int numDocumentsToIndex = 50 + random.nextInt(150); + AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); + Directory dir = newDirectory(); + MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()).setFlushPolicy(flushPolicy); + + final int numDWPT = 1 + random.nextInt(8); + DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool( + numDWPT); + iwc.setIndexerThreadPool(threadPool); + iwc.setMaxBufferedDocs(2 + random.nextInt(50)); + iwc.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); + IndexWriter writer = new IndexWriter(dir, iwc); + assertTrue(flushPolicy.flushOnDocCount()); + assertFalse(flushPolicy.flushOnDeleteTerms()); + assertFalse(flushPolicy.flushOnRAM()); + DocumentsWriter docsWriter = writer.getDocsWriter(); + assertNotNull(docsWriter); + DocumentsWriterFlushControl flushControl = docsWriter.flushControl; + assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes()); + + IndexThread[] threads = new IndexThread[numThreads[i]]; + for (int x = 0; x < threads.length; x++) { + threads[x] = new IndexThread(numDocs, numThreads[i], writer, + lineDocFile, false); + threads[x].start(); + } + + for (int x = 0; x < threads.length; x++) { + threads[x].join(); + } + + assertEquals(" all flushes must be due numThreads=" + numThreads[i], 0, + flushControl.flushBytes()); + assertEquals(numDocumentsToIndex, writer.numDocs()); + assertEquals(numDocumentsToIndex, writer.maxDoc()); + assertTrue("peak bytes without flush exceeded watermark", + flushPolicy.peakDocCountWithoutFlush <= iwc.getMaxBufferedDocs()); + assertActiveBytesAfter(flushControl); + writer.close(); + assertEquals(0, flushControl.activeBytes()); + dir.close(); + } + } + + public void testFlushPolicySetup() throws IOException { + Directory dir = newDirectory(); + FlushByRamOrCountsPolicy flushPolicy = new FlushByRamOrCountsPolicy(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()).setFlushPolicy(flushPolicy); + + final int numDWPT = 1 + random.nextInt(10); + DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool( + numDWPT); + iwc.setIndexerThreadPool(threadPool); + double maxMB = 1.0 + Math.ceil(random.nextDouble()); + iwc.setRAMBufferSizeMB(maxMB); + iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + IndexWriter writer = new IndexWriter(dir, iwc); + assertEquals((long) (maxMB * 1024. * 1024. * 2.), + flushPolicy.getMaxNetBytes()); + + writer.close(); + dir.close(); + } + + public void testRandom() throws IOException, InterruptedException { + final int numThreads = 1 + random.nextInt(8); + final int numDocumentsToIndex = 100 + random.nextInt(300); + AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()); + MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); + iwc.setFlushPolicy(flushPolicy); + + final int numDWPT = 1 + random.nextInt(8); + DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool( + numDWPT); + iwc.setIndexerThreadPool(threadPool); + + IndexWriter writer = new IndexWriter(dir, iwc); + DocumentsWriter docsWriter = writer.getDocsWriter(); + assertNotNull(docsWriter); + DocumentsWriterFlushControl flushControl = docsWriter.flushControl; + + assertEquals(" bytes must be 0 after init", 0, flushControl.flushBytes()); + + IndexThread[] threads = new IndexThread[numThreads]; + for (int x = 0; x < threads.length; x++) { + threads[x] = new IndexThread(numDocs, numThreads, writer, lineDocFile, + true); + threads[x].start(); + } + + for (int x = 0; x < threads.length; x++) { + threads[x].join(); + } + + assertEquals(" all flushes must be due", 0, flushControl.flushBytes()); + assertEquals(numDocumentsToIndex, writer.numDocs()); + assertEquals(numDocumentsToIndex, writer.maxDoc()); + if (flushPolicy.flushOnRAM() && !flushPolicy.flushOnDocCount() + && !flushPolicy.flushOnDeleteTerms()) { + final long maxRAMBytes = (long) (iwc.getRAMBufferSizeMB() * 1024. * 1024.); + assertTrue("peak bytes without flush exceeded watermark", + flushPolicy.peakBytesWithoutFlush <= maxRAMBytes); + if (flushPolicy.hasMarkedPending) { + assertTrue("max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, + maxRAMBytes <= flushControl.peakActiveBytes); + } + } + assertActiveBytesAfter(flushControl); + writer.commit(); + assertEquals(0, flushControl.activeBytes()); + IndexReader r = IndexReader.open(dir); + assertEquals(numDocumentsToIndex, r.numDocs()); + assertEquals(numDocumentsToIndex, r.maxDoc()); + if (!flushPolicy.flushOnRAM()) { + assertFalse("never stall if we don't flush on RAM", docsWriter.healthiness.wasStalled); + assertFalse("never block if we don't flush on RAM", docsWriter.healthiness.hasBlocked()); + } + r.close(); + writer.close(); + dir.close(); + } + + public void testHealthyness() throws InterruptedException, + CorruptIndexException, LockObtainFailedException, IOException { + + int[] numThreads = new int[] { 3 + random.nextInt(8), 1 }; + final int numDocumentsToIndex = 50 + random.nextInt(50); + for (int i = 0; i < numThreads.length; i++) { + AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); + MockDirectoryWrapper dir = newDirectory(); + // mock a very slow harddisk here so that flushing is very slow + dir.setThrottledIndexOutput(new ThrottledIndexOutput(ThrottledIndexOutput + .mBitsToBytes(50 + random.nextInt(10)), 5 + random.nextInt(5), null)); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()); + iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + iwc.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); + FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy(); + iwc.setFlushPolicy(flushPolicy); + + DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool( + numThreads[i]== 1 ? 1 : 2); + iwc.setIndexerThreadPool(threadPool); + // with such a small ram buffer we should be stalled quiet quickly + iwc.setRAMBufferSizeMB(0.25); + IndexWriter writer = new IndexWriter(dir, iwc); + IndexThread[] threads = new IndexThread[numThreads[i]]; + for (int x = 0; x < threads.length; x++) { + threads[x] = new IndexThread(numDocs, numThreads[i], writer, + lineDocFile, false); + threads[x].start(); + } + + for (int x = 0; x < threads.length; x++) { + threads[x].join(); + } + DocumentsWriter docsWriter = writer.getDocsWriter(); + assertNotNull(docsWriter); + DocumentsWriterFlushControl flushControl = docsWriter.flushControl; + assertEquals(" all flushes must be due", 0, flushControl.flushBytes()); + assertEquals(numDocumentsToIndex, writer.numDocs()); + assertEquals(numDocumentsToIndex, writer.maxDoc()); + if (flushControl.peakNetBytes > (long)(iwc.getRAMBufferSizeMB() * 1024d * 1024d * 2d)) { + assertTrue("should be unhealthy here numThreads: " + numThreads[i], + docsWriter.healthiness.wasStalled); + } + + if (numThreads[i] == 1) { // single thread could be unhealthy is a single + // doc is very large?! + assertFalse( + "single thread must not block numThreads: " + numThreads[i], + docsWriter.healthiness.hasBlocked()); + } else { + if (docsWriter.healthiness.wasStalled) { + // TODO maybe this assumtion is too strickt + assertTrue(" we should have blocked here numThreads: " + + numThreads[i], docsWriter.healthiness.hasBlocked()); + } + } + assertActiveBytesAfter(flushControl); + writer.close(true); + dir.close(); + } + } + + protected void assertActiveBytesAfter(DocumentsWriterFlushControl flushControl) { + Iterator allActiveThreads = flushControl.allActiveThreads(); + long bytesUsed = 0; + while (allActiveThreads.hasNext()) { + bytesUsed += allActiveThreads.next().perThread.bytesUsed(); + } + assertEquals(bytesUsed, flushControl.activeBytes()); + } + + public class IndexThread extends Thread { + IndexWriter writer; + IndexWriterConfig iwc; + LineFileDocs docs; + private AtomicInteger pendingDocs; + private final boolean doRandomCommit; + + public IndexThread(AtomicInteger pendingDocs, int numThreads, + IndexWriter writer, LineFileDocs docs, boolean doRandomCommit) { + this.pendingDocs = pendingDocs; + this.writer = writer; + iwc = writer.getConfig(); + this.docs = docs; + this.doRandomCommit = doRandomCommit; + } + + public void run() { + try { + long ramSize = 0; + while (pendingDocs.decrementAndGet() > -1) { + Document doc = docs.nextDoc(); + writer.addDocument(doc); + long newRamSize = writer.ramSizeInBytes(); + if (newRamSize != ramSize) { + ramSize = newRamSize; + } + if (doRandomCommit) { + int commit; + synchronized (random) { + commit = random.nextInt(20); + } + if (commit == 0) { + writer.commit(); + } + } + } + } catch (Throwable ex) { + throw new RuntimeException(ex); + } + } + } + + private static class MockDefaultFlushPolicy extends FlushByRamOrCountsPolicy { + long peakBytesWithoutFlush = Integer.MIN_VALUE; + long peakDocCountWithoutFlush = Integer.MIN_VALUE; + boolean hasMarkedPending = false; + + @Override + public void onDelete(DocumentsWriterFlushControl control, ThreadState state) { + final ArrayList pending = new ArrayList(); + final ArrayList notPending = new ArrayList(); + findPending(control, pending, notPending); + final boolean flushCurrent = state.flushPending; + final ThreadState toFlush; + if (state.flushPending) { + toFlush = state; + } else if (flushOnDeleteTerms() + && state.perThread.pendingDeletes.numTermDeletes.get() >= indexWriterConfig + .getMaxBufferedDeleteTerms()) { + toFlush = state; + } else { + toFlush = null; + } + super.onDelete(control, state); + if (toFlush != null) { + if (flushCurrent) { + assertTrue(pending.remove(toFlush)); + } else { + assertTrue(notPending.remove(toFlush)); + } + assertTrue(toFlush.flushPending); + hasMarkedPending = true; + } + + for (ThreadState threadState : notPending) { + assertFalse(threadState.flushPending); + } + } + + @Override + public void onInsert(DocumentsWriterFlushControl control, ThreadState state) { + final ArrayList pending = new ArrayList(); + final ArrayList notPending = new ArrayList(); + findPending(control, pending, notPending); + final boolean flushCurrent = state.flushPending; + long activeBytes = control.activeBytes(); + final ThreadState toFlush; + if (state.flushPending) { + toFlush = state; + } else if (flushOnDocCount() + && state.perThread.getNumDocsInRAM() >= indexWriterConfig + .getMaxBufferedDocs()) { + toFlush = state; + } else if (flushOnRAM() + && activeBytes >= (long) (indexWriterConfig.getRAMBufferSizeMB() * 1024. * 1024.)) { + toFlush = findLargestNonPendingWriter(control, state); + assertFalse(toFlush.flushPending); + } else { + toFlush = null; + } + super.onInsert(control, state); + if (toFlush != null) { + if (flushCurrent) { + assertTrue(pending.remove(toFlush)); + } else { + assertTrue(notPending.remove(toFlush)); + } + assertTrue(toFlush.flushPending); + hasMarkedPending = true; + } else { + peakBytesWithoutFlush = Math.max(activeBytes, peakBytesWithoutFlush); + peakDocCountWithoutFlush = Math.max(state.perThread.getNumDocsInRAM(), + peakDocCountWithoutFlush); + } + + for (ThreadState threadState : notPending) { + assertFalse(threadState.flushPending); + } + } + } + + static void findPending(DocumentsWriterFlushControl flushControl, + ArrayList pending, ArrayList notPending) { + Iterator allActiveThreads = flushControl.allActiveThreads(); + while (allActiveThreads.hasNext()) { + ThreadState next = allActiveThreads.next(); + if (next.flushPending) { + pending.add(next); + } else { + notPending.add(next); + } + } + } +} diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index 91e5491dca5..9911c107ab0 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -759,13 +759,14 @@ public class TestIndexWriter extends LuceneTestCase { writer.deleteDocuments(new Term("field", "aaa" + j)); _TestUtil.syncConcurrentMerges(writer); int flushCount = writer.getFlushCount(); + if (j == 1) lastFlushCount = flushCount; else if (j < 10) { // No new files should be created assertEquals(flushCount, lastFlushCount); } else if (10 == j) { - assertTrue(flushCount > lastFlushCount); + assertTrue("" + j, flushCount > lastFlushCount); lastFlushCount = flushCount; writer.getConfig().setRAMBufferSizeMB(0.000001); writer.getConfig().setMaxBufferedDeleteTerms(1); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java index 3052dab7fcf..8416c663267 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java @@ -70,6 +70,10 @@ public class TestIndexWriterConfig extends LuceneTestCase { assertEquals(IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR, conf.getReaderTermsIndexDivisor()); assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass()); assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass()); + assertNull(conf.getFlushPolicy()); + assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB()); + + // Sanity check - validate that all getters are covered. Set getters = new HashSet(); @@ -94,6 +98,9 @@ public class TestIndexWriterConfig extends LuceneTestCase { getters.add("getReaderPooling"); getters.add("getIndexerThreadPool"); getters.add("getReaderTermsIndexDivisor"); + getters.add("getFlushPolicy"); + getters.add("getRAMPerThreadHardLimitMB"); + for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { assertTrue("method " + m.getName() + " is not tested for defaults", getters.contains(m.getName())); @@ -241,6 +248,20 @@ public class TestIndexWriterConfig extends LuceneTestCase { // this is expected } + try { + conf.setRAMPerThreadHardLimitMB(2048); + fail("should not have succeeded to set RAMPerThreadHardLimitMB to >= 2048"); + } catch (IllegalArgumentException e) { + // this is expected + } + + try { + conf.setRAMPerThreadHardLimitMB(0); + fail("should not have succeeded to set RAMPerThreadHardLimitMB to 0"); + } catch (IllegalArgumentException e) { + // this is expected + } + assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates()); conf.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(5)); assertEquals(5, conf.getMaxThreadStates()); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index 6ab37b4e015..a71196ba883 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -124,7 +124,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { writer.close(); dir.close(); } - + // test when delete terms only apply to ram segments public void testRAMDeletes() throws IOException { for(int t=0;t<2;t++) { From a4c7a8883469250ff0ecd453f2cac68d6743b514 Mon Sep 17 00:00:00 2001 From: Shai Erera Date: Wed, 30 Mar 2011 18:47:57 +0000 Subject: [PATCH 038/200] LUCENE-3002: Add tests.iter.min to control tests.iter's behavior git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087036 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 7 ++++++- lucene/common-build.xml | 3 +++ .../org/apache/lucene/util/LuceneTestCase.java | 17 ++++++++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 448f11b0707..0221b958c3f 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -337,7 +337,6 @@ New features * LUCENE-3001: Added TrieFieldHelper to write solr compatible numeric fields without the solr dependency. (ryan) - Optimizations * LUCENE-2588: Don't store unnecessary suffixes when writing the terms @@ -374,6 +373,12 @@ Bug fixes indexes, causing existing deletions to be applied on the incoming indexes as well. (Shai Erera, Mike McCandless) +Test Cases + +* LUCENE-3002: added 'tests.iter.min' to control 'tests.iter' by allowing to + stop iterating if at least 'tests.iter.min' ran and a failure occured. + (Shai Erera, Chris Hostetter) + ======================= Lucene 3.x (not yet released) ======================= Changes in backwards compatibility policy diff --git a/lucene/common-build.xml b/lucene/common-build.xml index 38327fec496..7415bbcd916 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -78,6 +78,7 @@ + @@ -507,6 +508,8 @@ + + diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java index b56aaf2e4d5..65f731f005f 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java @@ -142,6 +142,8 @@ public abstract class LuceneTestCase extends Assert { public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random"); /** Get the number of times to run tests */ public static final int TEST_ITER = Integer.parseInt(System.getProperty("tests.iter", "1")); + /** Get the minimum number of times to run tests until a failure happens */ + public static final int TEST_ITER_MIN = Integer.parseInt(System.getProperty("tests.iter.min", Integer.toString(TEST_ITER))); /** Get the random seed for tests */ public static final String TEST_SEED = System.getProperty("tests.seed", "random"); /** whether or not nightly tests should run */ @@ -1208,11 +1210,24 @@ public abstract class LuceneTestCase extends Assert { if (VERBOSE) { System.out.println("\nNOTE: running test " + arg0.getName()); } + + // only print iteration info if the user requested more than one iterations + boolean verbose = VERBOSE && TEST_ITER > 1; + int lastIterFailed = -1; for (int i = 0; i < TEST_ITER; i++) { - if (VERBOSE && TEST_ITER > 1) { + if (verbose) { System.out.println("\nNOTE: running iter=" + (1+i) + " of " + TEST_ITER); } super.runChild(arg0, arg1); + if (testsFailed) { + lastIterFailed = i; + if (i == TEST_ITER_MIN - 1) { + if (verbose) { + System.out.println("\nNOTE: iteration " + lastIterFailed + " failed !"); + } + break; + } + } } } From 9fdc41f0f8250f275876630dc5731f9362b049a9 Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Wed, 30 Mar 2011 19:37:38 +0000 Subject: [PATCH 039/200] sync CHANGEs for 3.1 git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087056 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 179 +++++++++++++++++++------------------ lucene/contrib/CHANGES.txt | 85 +++++++++--------- solr/CHANGES.txt | 96 +++++++++++--------- 3 files changed, 191 insertions(+), 169 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 0221b958c3f..0ec3f79127e 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -393,7 +393,7 @@ Optimizations * LUCENE-2990: ArrayUtil/CollectionUtil.*Sort() methods now exit early on empty or one-element lists/arrays. (Uwe Schindler) -======================= Lucene 3.1 (not yet released) ======================= +======================= Lucene 3.1.0 ======================= Changes in backwards compatibility policy @@ -409,7 +409,7 @@ Changes in backwards compatibility policy * LUCENE-2190: Removed deprecated customScore() and customExplain() methods from experimental CustomScoreQuery. (Uwe Schindler) - + * LUCENE-2286: Enabled DefaultSimilarity.setDiscountOverlaps by default. This means that terms with a position increment gap of zero do not affect the norms calculation by default. (Robert Muir) @@ -447,10 +447,10 @@ Changes in backwards compatibility policy actual file's length if the file exists, and throws FileNotFoundException otherwise. Returning length=0 for a non-existent file is no longer allowed. If you relied on that, make sure to catch the exception. (Shai Erera) - + * LUCENE-2386: IndexWriter no longer performs an empty commit upon new index creation. Previously, if you passed an empty Directory and set OpenMode to - CREATE*, IndexWriter would make a first empty commit. If you need that + CREATE*, IndexWriter would make a first empty commit. If you need that behavior you can call writer.commit()/close() immediately after you create it. (Shai Erera, Mike McCandless) @@ -466,10 +466,10 @@ Changes in backwards compatibility policy values in multi-valued field has been changed for some cases in index. If you index empty fields and uses positions/offsets information on that fields, reindex is recommended. (David Smiley, Koji Sekiguchi) - + * LUCENE-2804: Directory.setLockFactory new declares throwing an IOException. (Shai Erera, Robert Muir) - + * LUCENE-2837: Added deprecations noting that in 4.0, Searcher and Searchable are collapsed into IndexSearcher; contrib/remote and MultiSearcher have been removed. (Mike McCandless) @@ -496,7 +496,7 @@ Changes in runtime behavior * LUCENE-2179: CharArraySet.clear() is now functional. (Robert Muir, Uwe Schindler) -* LUCENE-2455: IndexWriter.addIndexes no longer optimizes the target index +* LUCENE-2455: IndexWriter.addIndexes no longer optimizes the target index before it adds the new ones. Also, the existing segments are not merged and so the index will not end up with a single segment (unless it was empty before). In addition, addIndexesNoOptimize was renamed to addIndexes and no longer @@ -515,9 +515,9 @@ Changes in runtime behavior usage, allowing applications to accidentally open two writers on the same directory. (Mike McCandless) -* LUCENE-2701: maxMergeMBForOptimize and maxMergeDocs constraints set on - LogMergePolicy now affect optimize() as well (as opposed to only regular - merges). This means that you can run optimize() and too large segments won't +* LUCENE-2701: maxMergeMBForOptimize and maxMergeDocs constraints set on + LogMergePolicy now affect optimize() as well (as opposed to only regular + merges). This means that you can run optimize() and too large segments won't be merged. (Shai Erera) * LUCENE-2753: IndexReader and DirectoryReader .listCommits() now return a List, @@ -527,9 +527,9 @@ Changes in runtime behavior the IndexSearcher search methods that take an int nDocs will now throw IllegalArgumentException if nDocs is 0. Instead, you should use the newly added TotalHitCountCollector. (Mike McCandless) - -* LUCENE-2790: LogMergePolicy.useCompoundFile's logic now factors in noCFSRatio - to determine whether the passed in segment should be compound. + +* LUCENE-2790: LogMergePolicy.useCompoundFile's logic now factors in noCFSRatio + to determine whether the passed in segment should be compound. (Shai Erera, Earwin Burrfoot) * LUCENE-2805: IndexWriter now increments the index version on every change to @@ -549,7 +549,7 @@ Changes in runtime behavior * LUCENE-2010: Segments with 100% deleted documents are now removed on IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless) - + * LUCENE-2960: Allow some changes to IndexWriterConfig to take effect "live" (after an IW is instantiated), via IndexWriter.getConfig().setXXX(...) (Shay Banon, Mike McCandless) @@ -567,7 +567,7 @@ API Changes * LUCENE-2103: NoLockFactory should have a private constructor; until Lucene 4.0 the default one will be deprecated. - (Shai Erera via Uwe Schindler) + (Shai Erera via Uwe Schindler) * LUCENE-2177: Deprecate the Field ctors that take byte[] and Store. Since the removal of compressed fields, Store can only be YES, so @@ -587,30 +587,30 @@ API Changes files are no longer open by IndexReaders. (luocanrao via Mike McCandless) -* LUCENE-2282: IndexFileNames is exposed as a public class allowing for easier - use by external code. In addition it offers a matchExtension method which +* LUCENE-2282: IndexFileNames is exposed as a public class allowing for easier + use by external code. In addition it offers a matchExtension method which callers can use to query whether a certain file matches a certain extension. - (Shai Erera via Mike McCandless) + (Shai Erera via Mike McCandless) * LUCENE-124: Add a TopTermsBoostOnlyBooleanQueryRewrite to MultiTermQuery. This rewrite method is similar to TopTermsScoringBooleanQueryRewrite, but - only scores terms by their boost values. For example, this can be used - with FuzzyQuery to ensure that exact matches are always scored higher, + only scores terms by their boost values. For example, this can be used + with FuzzyQuery to ensure that exact matches are always scored higher, because only the boost will be used in scoring. (Robert Muir) - -* LUCENE-2015: Add a static method foldToASCII to ASCIIFoldingFilter to + +* LUCENE-2015: Add a static method foldToASCII to ASCIIFoldingFilter to expose its folding logic. (Cédrik Lime via Robert Muir) - -* LUCENE-2294: IndexWriter constructors have been deprecated in favor of a + +* LUCENE-2294: IndexWriter constructors have been deprecated in favor of a single ctor which accepts IndexWriterConfig and a Directory. You can set all - the parameters related to IndexWriter on IndexWriterConfig. The different - setter/getter methods were deprecated as well. One should call + the parameters related to IndexWriter on IndexWriterConfig. The different + setter/getter methods were deprecated as well. One should call writer.getConfig().getXYZ() to query for a parameter XYZ. - Additionally, the setter/getter related to MergePolicy were deprecated as + Additionally, the setter/getter related to MergePolicy were deprecated as well. One should interact with the MergePolicy directly. (Shai Erera via Mike McCandless) - -* LUCENE-2320: IndexWriter's MergePolicy configuration was moved to + +* LUCENE-2320: IndexWriter's MergePolicy configuration was moved to IndexWriterConfig and the respective methods on IndexWriter were deprecated. (Shai Erera via Mike McCandless) @@ -634,14 +634,14 @@ API Changes * LUCENE-2402: IndexWriter.deleteUnusedFiles now deletes unreferenced commit points too. If you use an IndexDeletionPolicy which holds onto index commits (such as SnapshotDeletionPolicy), you can call this method to remove those - commit points when they are not needed anymore (instead of waiting for the + commit points when they are not needed anymore (instead of waiting for the next commit). (Shai Erera) - + * LUCENE-2481: SnapshotDeletionPolicy.snapshot() and release() were replaced with equivalent ones that take a String (id) as argument. You can pass - whatever ID you want, as long as you use the same one when calling both. + whatever ID you want, as long as you use the same one when calling both. (Shai Erera) - + * LUCENE-2356: Add IndexWriterConfig.set/getReaderTermIndexDivisor, to set what IndexWriter passes for termsIndexDivisor to the readers it opens internally when apply deletions or creating a near-real-time @@ -651,7 +651,7 @@ API Changes in common/standard/ now implement the Word Break rules from the Unicode 6.0.0 Text Segmentation algorithm (UAX#29), covering the full range of Unicode code points, including values from U+FFFF to U+10FFFF - + ClassicTokenizer/Analyzer retains the old (pre-Lucene 3.1) StandardTokenizer/ Analyzer implementation and behavior. Only the Unicode Basic Multilingual Plane (code points from U+0000 to U+FFFF) is covered. @@ -659,16 +659,16 @@ API Changes UAX29URLEmailTokenizer tokenizes URLs and E-mail addresses according to the relevant RFCs, in addition to implementing the UAX#29 Word Break rules. (Steven Rowe, Robert Muir, Uwe Schindler) - + * LUCENE-2778: RAMDirectory now exposes newRAMFile() which allows to override and return a different RAMFile implementation. (Shai Erera) - + * LUCENE-2785: Added TotalHitCountCollector whose sole purpose is to count the number of hits matching the query. (Mike McCandless) -* LUCENE-2846: Deprecated IndexReader.setNorm(int, String, float). This method - is only syntactic sugar for setNorm(int, String, byte), but using the global - Similarity.getDefault().encodeNormValue(). Use the byte-based method instead +* LUCENE-2846: Deprecated IndexReader.setNorm(int, String, float). This method + is only syntactic sugar for setNorm(int, String, byte), but using the global + Similarity.getDefault().encodeNormValue(). Use the byte-based method instead to ensure that the norm is encoded with your Similarity. (Robert Muir, Mike McCandless) @@ -689,6 +689,9 @@ API Changes for AttributeImpls, but can still be provided (if needed). (Uwe Schindler) +* LUCENE-2691: Deprecate IndexWriter.getReader in favor of + IndexReader.open(IndexWriter) (Grant Ingersoll, Mike McCandless) + * LUCENE-2876: Deprecated Scorer.getSimilarity(). If your Scorer uses a Similarity, it should keep it itself. Fixed Scorers to pass their parent Weight, so that Scorer.visitSubScorers (LUCENE-2590) will work correctly. @@ -700,7 +703,7 @@ API Changes expert use cases can handle seeing deleted documents returned. The deletes remain buffered so that the next time you open an NRT reader and pass true, all deletes will be a applied. (Mike McCandless) - + * LUCENE-1253: LengthFilter (and Solr's KeepWordTokenFilter) now require up front specification of enablePositionIncrement. Together with StopFilter they have a common base class (FilteringTokenFilter) that handles @@ -711,7 +714,7 @@ Bug fixes * LUCENE-2249: ParallelMultiSearcher should shut down thread pool on close. (Martin Traverso via Uwe Schindler) - + * LUCENE-2273: FieldCacheImpl.getCacheEntries() used WeakHashMap incorrectly and lead to ConcurrentModificationException. (Uwe Schindler, Robert Muir) @@ -722,7 +725,7 @@ Bug fixes * LUCENE-2074: Reduce buffer size of lexer back to default on reset. (Ruben Laguna, Shai Erera via Uwe Schindler) - + * LUCENE-2496: Don't throw NPE if IndexWriter is opened with CREATE on a prior (corrupt) index missing its segments_N file. (Mike McCandless) @@ -731,10 +734,10 @@ Bug fixes assuming whitespace tokenization. Previously all CJK queries, for example, would be turned into phrase queries. The old behavior is preserved with the matchVersion parameter for previous versions. Additionally, you can - explicitly enable the old behavior with setAutoGeneratePhraseQueries(true) + explicitly enable the old behavior with setAutoGeneratePhraseQueries(true) (Robert Muir) - -* LUCENE-2537: FSDirectory.copy() implementation was unsafe and could result in + +* LUCENE-2537: FSDirectory.copy() implementation was unsafe and could result in OOM if a large file was copied. (Shai Erera) * LUCENE-2580: MultiPhraseQuery throws AIOOBE if number of positions @@ -752,14 +755,14 @@ Bug fixes * LUCENE-2802: NRT DirectoryReader returned incorrect values from getVersion, isOptimized, getCommitUserData, getIndexCommit and isCurrent due - to a mutable reference to the IndexWriters SegmentInfos. + to a mutable reference to the IndexWriters SegmentInfos. (Simon Willnauer, Earwin Burrfoot) * LUCENE-2852: Fixed corner case in RAMInputStream that would hit a false EOF after seeking to EOF then seeking back to same block you were just in and then calling readBytes (Robert Muir, Mike McCandless) -* LUCENE-2860: Fixed SegmentInfo.sizeInBytes to factor includeDocStores when it +* LUCENE-2860: Fixed SegmentInfo.sizeInBytes to factor includeDocStores when it decides whether to return the cached computed size or not. (Shai Erera) * LUCENE-2584: SegmentInfo.files() could hit ConcurrentModificationException if @@ -772,7 +775,7 @@ Bug fixes internally, it now calls Similarity.idfExplain(Collection, IndexSearcher). (Robert Muir) -* LUCENE-2693: RAM used by IndexWriter was slightly incorrectly computed. +* LUCENE-2693: RAM used by IndexWriter was slightly incorrectly computed. (Jason Rutherglen via Shai Erera) * LUCENE-1846: DateTools now uses the US locale everywhere, so DateTools.round() @@ -788,6 +791,9 @@ Bug fixes been rounded down to 0 instead of being rounded up to the smallest positive number. (yonik) +* LUCENE-2936: PhraseQuery score explanations were not correctly + identifying matches vs non-matches. (hossman) + * LUCENE-2975: A hotspot bug corrupts IndexInput#readVInt()/readVLong() if the underlying readByte() is inlined (which happens e.g. in MMapDirectory). The loop was unwinded which makes the hotspot bug disappear. @@ -796,30 +802,30 @@ Bug fixes New features * LUCENE-2128: Parallelized fetching document frequencies during weight - creation. (Israel Tsadok, Simon Willnauer via Uwe Schindler) + creation. (Israel Tsadok, Simon Willnauer via Uwe Schindler) * LUCENE-2069: Added Unicode 4 support to CharArraySet. Due to the switch to Java 5, supplementary characters are now lowercased correctly if the set is created as case insensitive. - CharArraySet now requires a Version argument to preserve - backwards compatibility. If Version < 3.1 is passed to the constructor, + CharArraySet now requires a Version argument to preserve + backwards compatibility. If Version < 3.1 is passed to the constructor, CharArraySet yields the old behavior. (Simon Willnauer) - + * LUCENE-2069: Added Unicode 4 support to LowerCaseFilter. Due to the switch to Java 5, supplementary characters are now lowercased correctly. - LowerCaseFilter now requires a Version argument to preserve - backwards compatibility. If Version < 3.1 is passed to the constructor, - LowerCaseFilter yields the old behavior. (Simon Willnauer, Robert Muir) + LowerCaseFilter now requires a Version argument to preserve + backwards compatibility. If Version < 3.1 is passed to the constructor, + LowerCaseFilter yields the old behavior. (Simon Willnauer, Robert Muir) * LUCENE-2034: Added ReusableAnalyzerBase, an abstract subclass of Analyzer that makes it easier to reuse TokenStreams correctly. This issue also added StopwordAnalyzerBase, which improves consistency of all Analyzers that use - stopwords, and implement many analyzers in contrib with it. + stopwords, and implement many analyzers in contrib with it. (Simon Willnauer via Robert Muir) - + * LUCENE-2198, LUCENE-2901: Support protected words in stemming TokenFilters using a new KeywordAttribute. (Simon Willnauer, Drew Farris via Uwe Schindler) - + * LUCENE-2183, LUCENE-2240, LUCENE-2241: Added Unicode 4 support to CharTokenizer and its subclasses. CharTokenizer now has new int-API which is conditionally preferred to the old char-API depending @@ -828,8 +834,8 @@ New features * LUCENE-2247: Added a CharArrayMap for performance improvements in some stemmers and synonym filters. (Uwe Schindler) - -* LUCENE-2320: Added SetOnce which wraps an object and allows it to be set + +* LUCENE-2320: Added SetOnce which wraps an object and allows it to be set exactly once. (Shai Erera via Mike McCandless) * LUCENE-2314: Added AttributeSource.copyTo(AttributeSource) that @@ -856,19 +862,19 @@ New features Directory.copyTo, and use nio's FileChannel.transferTo when copying files between FSDirectory instances. (Earwin Burrfoot via Mike McCandless). - + * LUCENE-2074: Make StandardTokenizer fit for Unicode 4.0, if the matchVersion parameter is Version.LUCENE_31. (Uwe Schindler) * LUCENE-2385: Moved NoDeletionPolicy from benchmark to core. NoDeletionPolicy can be used to prevent commits from ever getting deleted from the index. (Shai Erera) - -* LUCENE-1585: IndexWriter now accepts a PayloadProcessorProvider which can - return a DirPayloadProcessor for a given Directory, which returns a - PayloadProcessor for a given Term. The PayloadProcessor will be used to + +* LUCENE-1585: IndexWriter now accepts a PayloadProcessorProvider which can + return a DirPayloadProcessor for a given Directory, which returns a + PayloadProcessor for a given Term. The PayloadProcessor will be used to process the payloads of the segments as they are merged (e.g. if one wants to - rewrite payloads of external indexes as they are added, or of local ones). + rewrite payloads of external indexes as they are added, or of local ones). (Shai Erera, Michael Busch, Mike McCandless) * LUCENE-2440: Add support for custom ExecutorService in @@ -881,7 +887,7 @@ New features * LUCENE-2526: Don't throw NPE from MultiPhraseQuery.toString when it's empty. (Ross Woolf via Mike McCandless) - + * LUCENE-2559: Added SegmentReader.reopen methods (John Wang via Mike McCandless) @@ -897,17 +903,20 @@ New features to add span support: SpanMultiTermQueryWrapper. Using this wrapper its easy to add fuzzy/wildcard to e.g. a SpanNearQuery. (Robert Muir, Uwe Schindler) - + * LUCENE-2838: ConstantScoreQuery now directly supports wrapping a Query instance for stripping off scores. The use of a QueryWrapperFilter is no longer needed and discouraged for that use case. Directly wrapping Query improves performance, as out-of-order collection is now supported. (Uwe Schindler) -* LUCENE-2864: Add getMaxTermFrequency (maximum within-document TF) to +* LUCENE-2864: Add getMaxTermFrequency (maximum within-document TF) to FieldInvertState so that it can be used in Similarity.computeNorm. (Robert Muir) +* LUCENE-2720: Segments now record the code version which created them. + (Shai Erera, Mike McCandless, Uwe Schindler) + * LUCENE-2474: Added expert ReaderFinishedListener API to IndexReader, to allow apps that maintain external per-segment caches to evict entries when a segment is finished. (Shay Banon, Yonik @@ -916,8 +925,8 @@ New features * LUCENE-2911: The new StandardTokenizer, UAX29URLEmailTokenizer, and the ICUTokenizer in contrib now all tag types with a consistent set of token types (defined in StandardTokenizer). Tokens in the major - CJK types are explicitly marked to allow for custom downstream handling: - , , , and . + CJK types are explicitly marked to allow for custom downstream handling: + , , , and . (Robert Muir, Steven Rowe) * LUCENE-2913: Add missing getters to Numeric* classes. (Uwe Schindler) @@ -942,7 +951,7 @@ Optimizations * LUCENE-2137: Switch to AtomicInteger for some ref counting (Earwin Burrfoot via Mike McCandless) -* LUCENE-2123, LUCENE-2261: Move FuzzyQuery rewrite to separate RewriteMode +* LUCENE-2123, LUCENE-2261: Move FuzzyQuery rewrite to separate RewriteMode into MultiTermQuery. The number of fuzzy expansions can be specified with the maxExpansions parameter to FuzzyQuery. (Uwe Schindler, Robert Muir, Mike McCandless) @@ -976,12 +985,12 @@ Optimizations TermAttributeImpl, move DEFAULT_TYPE constant to TypeInterface, improve null-handling for TypeAttribute. (Uwe Schindler) -* LUCENE-2329: Switch TermsHash* from using a PostingList object per unique +* LUCENE-2329: Switch TermsHash* from using a PostingList object per unique term to parallel arrays, indexed by termID. This reduces garbage collection overhead significantly, which results in great indexing performance wins when the available JVM heap space is low. This will become even more important when the DocumentsWriter RAM buffer is searchable in the future, - because then it will make sense to make the RAM buffers as large as + because then it will make sense to make the RAM buffers as large as possible. (Mike McCandless, Michael Busch) * LUCENE-2380: The terms field cache methods (getTerms, @@ -996,7 +1005,7 @@ Optimizations causing too many fallbacks to compare-by-value (instead of by-ord). (Mike McCandless) -* LUCENE-2574: IndexInput exposes copyBytes(IndexOutput, long) to allow for +* LUCENE-2574: IndexInput exposes copyBytes(IndexOutput, long) to allow for efficient copying by sub-classes. Optimized copy is implemented for RAM and FS streams. (Shai Erera) @@ -1019,15 +1028,15 @@ Optimizations * LUCENE-2010: Segments with 100% deleted documents are now removed on IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless) - + * LUCENE-1472: Removed synchronization from static DateTools methods by using a ThreadLocal. Also converted DateTools.Resolution to a Java 5 enum (this should not break backwards). (Uwe Schindler) Build -* LUCENE-2124: Moved the JDK-based collation support from contrib/collation - into core, and moved the ICU-based collation support into contrib/icu. +* LUCENE-2124: Moved the JDK-based collation support from contrib/collation + into core, and moved the ICU-based collation support into contrib/icu. (Robert Muir) * LUCENE-2326: Removed SVN checkouts for backwards tests. The backwards @@ -1039,14 +1048,14 @@ Build * LUCENE-1709: Tests are now parallelized by default (except for benchmark). You can force them to run sequentially by passing -Drunsequential=1 on the command - line. The number of threads that are spawned per CPU defaults to '1'. If you + line. The number of threads that are spawned per CPU defaults to '1'. If you wish to change that, you can run the tests with -DthreadsPerProcessor=[num]. (Robert Muir, Shai Erera, Peter Kofler) * LUCENE-2516: Backwards tests are now compiled against released lucene-core.jar from tarball of previous version. Backwards tests are now packaged together with src distribution. (Uwe Schindler) - + * LUCENE-2611: Added Ant target to install IntelliJ IDEA configuration: "ant idea". See http://wiki.apache.org/lucene-java/HowtoConfigureIntelliJ (Steven Rowe) @@ -1055,8 +1064,8 @@ Build generating Maven artifacts (Steven Rowe) * LUCENE-2609: Added jar-test-framework Ant target which packages Lucene's - tests' framework classes. (Drew Farris, Grant Ingersoll, Shai Erera, Steven - Rowe) + tests' framework classes. (Drew Farris, Grant Ingersoll, Shai Erera, + Steven Rowe) Test Cases @@ -1092,18 +1101,18 @@ Test Cases access to "real" files from the test folder itself, can use LuceneTestCase(J4).getDataFile(). (Uwe Schindler) -* LUCENE-2398, LUCENE-2611: Improve tests to work better from IDEs such +* LUCENE-2398, LUCENE-2611: Improve tests to work better from IDEs such as Eclipse and IntelliJ. (Paolo Castagna, Steven Rowe via Robert Muir) * LUCENE-2804: add newFSDirectory to LuceneTestCase to create a FSDirectory at random. (Shai Erera, Robert Muir) - + Documentation * LUCENE-2579: Fix oal.search's package.html description of abstract methods. (Santiago M. Mola via Mike McCandless) - + * LUCENE-2625: Add a note to IndexReader.termDocs() with additional verbiage that the TermEnum must be seeked since it is unpositioned. (Adriano Crestani via Robert Muir) diff --git a/lucene/contrib/CHANGES.txt b/lucene/contrib/CHANGES.txt index cbf58336eb7..a4a474b512a 100644 --- a/lucene/contrib/CHANGES.txt +++ b/lucene/contrib/CHANGES.txt @@ -47,26 +47,26 @@ API Changes (No changes) -======================= Lucene 3.1 (not yet released) ======================= +======================= Lucene 3.1.0 ======================= Changes in backwards compatibility policy * LUCENE-2100: All Analyzers in Lucene-contrib have been marked as final. Analyzers should be only act as a composition of TokenStreams, users should compose their own analyzers instead of subclassing existing ones. - (Simon Willnauer) + (Simon Willnauer) * LUCENE-2194, LUCENE-2201: Snowball APIs were upgraded to snowball revision - 502 (with some local modifications for improved performance). - Index backwards compatibility and binary backwards compatibility is - preserved, but some protected/public member variables changed type. This - does NOT affect java code/class files produced by the snowball compiler, + 502 (with some local modifications for improved performance). + Index backwards compatibility and binary backwards compatibility is + preserved, but some protected/public member variables changed type. This + does NOT affect java code/class files produced by the snowball compiler, but technically is a backwards compatibility break. (Robert Muir) - + * LUCENE-2226: Moved contrib/snowball functionality into contrib/analyzers. Be sure to remove any old obselete lucene-snowball jar files from your classpath! (Robert Muir) - + * LUCENE-2323: Moved contrib/wikipedia functionality into contrib/analyzers. Additionally the package was changed from org.apache.lucene.wikipedia.analysis to org.apache.lucene.analysis.wikipedia. (Robert Muir) @@ -74,30 +74,30 @@ Changes in backwards compatibility policy * LUCENE-2581: Added new methods to FragmentsBuilder interface. These methods are used to set pre/post tags and Encoder. (Koji Sekiguchi) - * LUCENE-2391: Improved spellchecker (re)build time/ram usage by omitting + * LUCENE-2391: Improved spellchecker (re)build time/ram usage by omitting frequencies/positions/norms for single-valued fields, modifying the default ramBufferMBSize to match IndexWriterConfig (16MB), making index optimization an optional boolean parameter, and modifying the incremental update logic - to work well with unoptimized spellcheck indexes. The indexDictionary() methods - were made final to ensure a hard backwards break in case you were subclassing + to work well with unoptimized spellcheck indexes. The indexDictionary() methods + were made final to ensure a hard backwards break in case you were subclassing Spellchecker. In general, subclassing Spellchecker is not recommended. (Robert Muir) - + Changes in runtime behavior * LUCENE-2117: SnowballAnalyzer uses TurkishLowerCaseFilter instead of LowercaseFilter to correctly handle the unique Turkish casing behavior if used with Version > 3.0 and the TurkishStemmer. - (Robert Muir via Simon Willnauer) + (Robert Muir via Simon Willnauer) - * LUCENE-2055: GermanAnalyzer now uses the Snowball German2 algorithm and + * LUCENE-2055: GermanAnalyzer now uses the Snowball German2 algorithm and stopwords list by default for Version > 3.0. (Robert Muir, Uwe Schindler, Simon Willnauer) Bug fixes - * LUCENE-2855: contrib queryparser was using CharSequence as key in some internal - Map instances, which was leading to incorrect behaviour, since some CharSequence - implementors do not override hashcode and equals methods. Now the internal Maps + * LUCENE-2855: contrib queryparser was using CharSequence as key in some internal + Map instances, which was leading to incorrect behavior, since some CharSequence + implementors do not override hashcode and equals methods. Now the internal Maps are using String instead. (Adriano Crestani) * LUCENE-2068: Fixed ReverseStringFilter which was not aware of supplementary @@ -106,9 +106,9 @@ Bug fixes now reverses supplementary characters correctly if used with Version > 3.0. (Simon Willnauer, Robert Muir) - * LUCENE-2035: TokenSources.getTokenStream() does not assign positionIncrement. + * LUCENE-2035: TokenSources.getTokenStream() does not assign positionIncrement. (Christopher Morris via Mark Miller) - + * LUCENE-2055: Deprecated RussianTokenizer, RussianStemmer, RussianStemFilter, FrenchStemmer, FrenchStemFilter, DutchStemmer, and DutchStemFilter. For these Analyzers, SnowballFilter is used instead (for Version > 3.0), as @@ -118,7 +118,7 @@ Bug fixes * LUCENE-2184: Fixed bug with handling best fit value when the proper best fit value is not an indexed field. Note, this change affects the APIs. (Grant Ingersoll) - + * LUCENE-2359: Fix bug in CartesianPolyFilterBuilder related to handling of behavior around the 180th meridian (Grant Ingersoll) @@ -135,15 +135,15 @@ Bug fixes and regenerating a new .nrm with 'ant gennorm2'. (David Bowen via Robert Muir) * LUCENE-2653: ThaiWordFilter depends on the JRE having a Thai dictionary, which is not - always the case. If the dictionary is unavailable, the filter will now throw + always the case. If the dictionary is unavailable, the filter will now throw UnsupportedOperationException in the constructor. (Robert Muir) - * LUCENE-589: Fix contrib/demo for international documents. + * LUCENE-589: Fix contrib/demo for international documents. (Curtis d'Entremont via Robert Muir) - + * LUCENE-2246: Fix contrib/demo for Turkish html documents. - (Selim Nadi via Robert Muir) - + (Selim Nadi via Robert Muir) + * LUCENE-590: Demo HTML parser gives incorrect summaries when title is repeated as a heading (Curtis d'Entremont via Robert Muir) @@ -153,9 +153,9 @@ Bug fixes * LUCENE-2874: Highlighting overlapping tokens outputted doubled words. (Pierre Gossé via Robert Muir) - * LUCENE-2943: Fix thread-safety issues with ICUCollationKeyFilter. + * LUCENE-2943: Fix thread-safety issues with ICUCollationKeyFilter. (Robert Muir) - + API Changes * LUCENE-2867: Some contrib queryparser methods that receives CharSequence as @@ -165,7 +165,7 @@ API Changes * LUCENE-2147: Spatial GeoHashUtils now always decode GeoHash strings with full precision. GeoHash#decode_exactly(String) was merged into GeoHash#decode(String). (Chris Male, Simon Willnauer) - + * LUCENE-2204: Change some package private classes/members to publicly accessible to implement custom FragmentsBuilders. (Koji Sekiguchi) @@ -182,14 +182,14 @@ API Changes * LUCENE-2626: FastVectorHighlighter: enable FragListBuilder and FragmentsBuilder to be set per-field override. (Koji Sekiguchi) - * LUCENE-2712: FieldBoostMapAttribute in contrib/queryparser was changed from + * LUCENE-2712: FieldBoostMapAttribute in contrib/queryparser was changed from a Map to a Map. Per the CharSequence javadoc, CharSequence is inappropriate as a map key. (Robert Muir) * LUCENE-1937: Add more methods to manipulate QueryNodeProcessorPipeline elements. QueryNodeProcessorPipeline now implements the List interface, this is useful if you want to extend or modify an existing pipeline. (Adriano Crestani via Robert Muir) - + * LUCENE-2754, LUCENE-2757: Deprecated SpanRegexQuery. Use new SpanMultiTermQueryWrapper(new RegexQuery()) instead. (Robert Muir, Uwe Schindler) @@ -199,10 +199,10 @@ API Changes * LUCENE-2830: Use StringBuilder instead of StringBuffer across Benchmark, and remove the StringBuffer HtmlParser.parse() variant. (Shai Erera) - + * LUCENE-2920: Deprecated ShingleMatrixFilter as it is unmaintained and does not work with custom Attributes or custom payload encoders. (Uwe Schindler) - + New features * LUCENE-2500: Added DirectIOLinuxDirectory, a Linux-specific @@ -210,14 +210,14 @@ New features cache. This is useful to prevent segment merging from evicting pages from the buffer cache, since fadvise/madvise do not seem. (Michael McCandless) - + * LUCENE-2306: Add NumericRangeFilter and NumericRangeQuery support to XMLQueryParser. (Jingkei Ly, via Mark Harwood) * LUCENE-2102: Add a Turkish LowerCase Filter. TurkishLowerCaseFilter handles Turkish and Azeri unique casing behavior correctly. (Ahmet Arslan, Robert Muir via Simon Willnauer) - + * LUCENE-2039: Add a extensible query parser to contrib/misc. ExtendableQueryParser enables arbitrary parser extensions based on a customizable field naming scheme. @@ -225,11 +225,11 @@ New features * LUCENE-2067: Add a Czech light stemmer. CzechAnalyzer will now stem words when Version is set to 3.1 or higher. (Robert Muir) - + * LUCENE-2062: Add a Bulgarian analyzer. (Robert Muir, Simon Willnauer) * LUCENE-2206: Add Snowball's stopword lists for Danish, Dutch, English, - Finnish, French, German, Hungarian, Italian, Norwegian, Russian, Spanish, + Finnish, French, German, Hungarian, Italian, Norwegian, Russian, Spanish, and Swedish. These can be loaded with WordListLoader.getSnowballWordSet. (Robert Muir, Simon Willnauer) @@ -237,7 +237,7 @@ New features (Koji Sekiguchi) * LUCENE-2218: ShingleFilter supports minimum shingle size, and the separator - character is now configurable. Its also up to 20% faster. + character is now configurable. Its also up to 20% faster. (Steven Rowe via Robert Muir) * LUCENE-2234: Add a Hindi analyzer. (Robert Muir) @@ -267,7 +267,7 @@ New features * LUCENE-2298: Add analyzers/stempel, an algorithmic stemmer with support for the Polish language. (Andrzej Bialecki via Robert Muir) - * LUCENE-2400: ShingleFilter was changed to don't output all-filler shingles and + * LUCENE-2400: ShingleFilter was changed to don't output all-filler shingles and unigrams, and uses a more performant algorithm to build grams using a linked list of AttributeSource.cloneAttributes() instances and the new copyTo() method. (Steven Rowe via Uwe Schindler) @@ -286,7 +286,7 @@ New features * LUCENE-2464: FastVectorHighlighter: add SingleFragListBuilder to return entire field contents. (Koji Sekiguchi) - * LUCENE-2503: Added lighter stemming alternatives for European languages. + * LUCENE-2503: Added lighter stemming alternatives for European languages. (Robert Muir) * LUCENE-2581: FastVectorHighlighter: add Encoder to FragmentsBuilder. @@ -294,20 +294,23 @@ New features * LUCENE-2624: Add Analyzers for Armenian, Basque, and Catalan, from snowball. (Robert Muir) - + * LUCENE-1938: PrecedenceQueryParser is now implemented with the flexible QP framework. This means that you can also add this functionality to your own QP pipeline by using BooleanModifiersQueryNodeProcessor, for example instead of GroupQueryNodeProcessor. (Adriano Crestani via Robert Muir) * LUCENE-2791: Added WindowsDirectory, a Windows-specific Directory impl - that doesn't synchronize on the file handle. This can be useful to + that doesn't synchronize on the file handle. This can be useful to avoid the performance problems of SimpleFSDirectory and NIOFSDirectory. (Robert Muir, Simon Willnauer, Uwe Schindler, Michael McCandless) * LUCENE-2842: Add analyzer for Galician. Also adds the RSLP (Orengo) stemmer for Portuguese. (Robert Muir) + * SOLR-1057: Add PathHierarchyTokenizer that represents file path hierarchies as synonyms of + /something, /something/something, /something/something/else. (Ryan McKinley, Koji Sekiguchi) + Build * LUCENE-2124: Moved the JDK-based collation support from contrib/collation diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d6e9b20d24c..0f36491e0be 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -247,24 +247,26 @@ Documentation ---------------------- -================== 3.1.0-dev ================== +================== 3.1.0 ================== Versions of Major Components --------------------- -Apache Lucene trunk +Apache Lucene 3.1.0 Apache Tika 0.8 Carrot2 3.4.2 +Velocity 1.6.1 and Velocity Tools 2.0-beta3 +Apache UIMA 2.3.1-SNAPSHOT Upgrading from Solr 1.4 ---------------------- -* The Lucene index format has changed and as a result, once you upgrade, +* The Lucene index format has changed and as a result, once you upgrade, previous versions of Solr will no longer be able to read your indices. In a master/slave configuration, all searchers/slaves should be upgraded before the master. If the master were to be updated first, the older searchers would not be able to read the new index format. -* The Solr JavaBin format has changed as of Solr 3.1. If you are using the +* The Solr JavaBin format has changed as of Solr 3.1. If you are using the JavaBin format, you will need to upgrade your SolrJ client. (SOLR-2034) * The experimental ALIAS command has been removed (SOLR-1637) @@ -275,10 +277,10 @@ Upgrading from Solr 1.4 is deprecated (SOLR-1696) * The deprecated HTMLStripReader, HTMLStripWhitespaceTokenizerFactory and - HTMLStripStandardTokenizerFactory were removed. To strip HTML tags, - HTMLStripCharFilter should be used instead, and it works with any + HTMLStripStandardTokenizerFactory were removed. To strip HTML tags, + HTMLStripCharFilter should be used instead, and it works with any Tokenizer of your choice. (SOLR-1657) - + * Field compression is no longer supported. Fields that were formerly compressed will be uncompressed as index segments are merged. For shorter fields, this may actually be an improvement, as the compression @@ -287,24 +289,24 @@ Upgrading from Solr 1.4 * SOLR-1845: The TermsComponent response format was changed so that the "terms" container is a map instead of a named list. This affects response formats like JSON, but not XML. (yonik) - + * SOLR-1876: All Analyzers and TokenStreams are now final to enforce the decorator pattern. (rmuir, uschindler) -* LUCENE-2608: Added the ability to specify the accuracy on a per request basis. +* LUCENE-2608: Added the ability to specify the accuracy on a per request basis. It is recommended that implementations of SolrSpellChecker should change over to the new SolrSpellChecker methods using the new SpellingOptions class, but are not required to. While this change is backward compatible, the trunk version of Solr has already dropped support for all but the SpellingOptions method. (gsingers) * readercycle script was removed. (SOLR-2046) -* In previous releases, sorting or evaluating function queries on +* In previous releases, sorting or evaluating function queries on fields that were "multiValued" (either by explicit declaration in schema.xml or by implict behavior because the "version" attribute on the schema was less then 1.2) did not generally work, but it would sometimes silently act as if it succeeded and order the docs arbitrarily. Solr will now fail on any attempt to sort, or apply a - function to, multi-valued fields + function to, multi-valued fields * The DataImportHandler jars are no longer included in the solr WAR and should be added in Solr's lib directory, or referenced @@ -374,13 +376,13 @@ New Features * SOLR-1379: Add RAMDirectoryFactory for non-persistent in memory index storage. (Alex Baranov via yonik) -* SOLR-1857: Synced Solr analysis with Lucene 3.1. Added KeywordMarkerFilterFactory - and StemmerOverrideFilterFactory, which can be used to tune stemming algorithms. +* SOLR-1857: Synced Solr analysis with Lucene 3.1. Added KeywordMarkerFilterFactory + and StemmerOverrideFilterFactory, which can be used to tune stemming algorithms. Added factories for Bulgarian, Czech, Hindi, Turkish, and Wikipedia analysis. Improved the performance of SnowballPorterFilterFactory. (rmuir) -* SOLR-1657: Converted remaining TokenStreams to the Attributes-based API. All Solr - TokenFilters now support custom Attributes, and some have improved performance: +* SOLR-1657: Converted remaining TokenStreams to the Attributes-based API. All Solr + TokenFilters now support custom Attributes, and some have improved performance: especially WordDelimiterFilter and CommonGramsFilter. (rmuir, cmale, uschindler) * SOLR-1740: ShingleFilterFactory supports the "minShingleSize" and "tokenSeparator" @@ -389,10 +391,10 @@ New Features * SOLR-744: ShingleFilterFactory supports the "outputUnigramsIfNoShingles" parameter, to output unigrams if the number of input tokens is fewer than - minShingleSize, and no shingles can be generated. + minShingleSize, and no shingles can be generated. (Chris Harris via Steven Rowe) -* SOLR-1923: PhoneticFilterFactory now has support for the +* SOLR-1923: PhoneticFilterFactory now has support for the Caverphone algorithm. (rmuir) * SOLR-1957: The VelocityResponseWriter contrib moved to core. @@ -460,7 +462,7 @@ New Features (Ankul Garg, Jason Rutherglen, Shalin Shekhar Mangar, Grant Ingersoll, Robert Muir, ab) * SOLR-1568: Added "native" filtering support for PointType, GeohashField. Added LatLonType with filtering support too. See - http://wiki.apache.org/solr/SpatialSearch and the example. Refactored some items in Lucene spatial. + http://wiki.apache.org/solr/SpatialSearch and the example. Refactored some items in Lucene spatial. Removed SpatialTileField as the underlying CartesianTier is broken beyond repair and is going to be moved. (gsingers) * SOLR-2128: Full parameter substitution for function queries. @@ -515,7 +517,7 @@ Optimizations Bug Fixes ---------------------- -* SOLR-1769: Solr 1.4 Replication - Repeater throwing NullPointerException (Jörgen Rydenius via noble) +* SOLR-1769: Solr 1.4 Replication - Repeater throwing NullPointerException (Jörgen Rydenius via noble) * SOLR-1432: Make the new ValueSource.getValues(context,reader) delegate to the original ValueSource.getValues(reader) so custom sources @@ -538,8 +540,8 @@ Bug Fixes * SOLR-1584: SolrJ - SolrQuery.setIncludeScore() incorrectly added fl=score to the parameter list instead of appending score to the existing field list. (yonik) - -* SOLR-1580: Solr Configuration ignores 'mergeFactor' parameter, always + +* SOLR-1580: Solr Configuration ignores 'mergeFactor' parameter, always uses Lucene default. (Lance Norskog via Mark Miller) * SOLR-1593: ReverseWildcardFilter didn't work for surrogate pairs @@ -556,7 +558,7 @@ Bug Fixes set when streaming updates, rather than using UTF-8 as the HTTP headers indicated, leading to an encoding mismatch. (hossman, yonik) -* SOLR-1587: A distributed search request with fl=score, didn't match +* SOLR-1587: A distributed search request with fl=score, didn't match the behavior of a non-distributed request since it only returned the id,score fields instead of all fields in addition to score. (yonik) @@ -565,7 +567,7 @@ Bug Fixes * SOLR-1615: Backslash escaping did not work in quoted strings for local param arguments. (Wojtek Piaseczny, yonik) -* SOLR-1628: log contains incorrect number of adds and deletes. +* SOLR-1628: log contains incorrect number of adds and deletes. (Thijs Vonk via yonik) * SOLR-343: Date faceting now respects facet.mincount limiting @@ -593,7 +595,7 @@ Bug Fixes (never officially released) introduced another hanging bug due to connections not being released. (Attila Babo, Erik Hetzner, Johannes Tuchscherer via yonik) - + * SOLR-1748, SOLR-1747, SOLR-1746, SOLR-1745, SOLR-1744: Streams and Readers retrieved from ContentStreams are not closed in various places, resulting in file descriptor leaks. @@ -602,7 +604,7 @@ Bug Fixes * SOLR-1753: StatsComponent throws NPE when getting statistics for facets in distributed search (Janne Majaranta via koji) -* SOLR-1736:In the slave , If 'mov'ing file does not succeed , copy the file (noble) +* SOLR-1736:In the slave , If 'mov'ing file does not succeed , copy the file (noble) * SOLR-1579: Fixes to XML escaping in stats.jsp (David Bowen and hossman) @@ -656,7 +658,7 @@ Bug Fixes * SOLR-2047: ReplicationHandler should accept bool type for enable flag. (koji) -* SOLR-1630: Fix spell checking collation issue related to token positions (rmuir, gsingers) +* SOLR-1630: Fix spell checking collation issue related to token positions (rmuir, gsingers) * SOLR-2100: The replication handler backup command didn't save the commit point and hence could fail when a newer commit caused the older commit point @@ -665,7 +667,7 @@ Bug Fixes * SOLR-2114: Fixed parsing error in hsin function. The function signature has changed slightly. (gsingers) -* SOLR-2083: SpellCheckComponent misreports suggestions when distributed (James Dyer via gsingers) +* SOLR-2083: SpellCheckComponent misreports suggestions when distributed (James Dyer via gsingers) * SOLR-2111: Change exception handling in distributed faceting to work more like non-distributed faceting, change facet_counts/exception from a String @@ -689,9 +691,9 @@ Bug Fixes * SOLR-2173: Suggester should always rebuild Lookup data if Lookup.load fails. (ab) * SOLR-2081: BaseResponseWriter.isStreamingDocs causes - SingleResponseWriter.end to be called 2x - (Chris A. Mattmann via hossman) - + SingleResponseWriter.end to be called 2x + (Chris A. Mattmann via hossman) + * SOLR-2219: The init() method of every SolrRequestHandler was being called twice. (ambikeshwar singh and hossman) @@ -716,7 +718,7 @@ Bug Fixes * SOLR-482: Provide more exception handling in CSVLoader (gsingers) -* SOLR-1283: HTMLStripCharFilter sometimes threw a "Mark Invalid" exception. +* SOLR-1283: HTMLStripCharFilter sometimes threw a "Mark Invalid" exception. (Julien Coloos, hossman, yonik) * SOLR-2085: Improve SolrJ behavior when FacetComponent comes before @@ -743,21 +745,29 @@ Bug Fixes * SOLR-2380: Distributed faceting could miss values when facet.sort=index and when facet.offset was greater than 0. (yonik) - + * SOLR-1656: XIncludes and other HREFs in XML files loaded by ResourceLoader - are fixed to be resolved using the URI standard (RFC 2396). The system - identifier is no longer a plain filename with path, it gets initialized - using a custom URI scheme "solrres:". This scheme is resolved using a - EntityResolver that utilizes ResourceLoader - (org.apache.solr.common.util.SystemIdResolver). This makes all relative - pathes in Solr's config files behave like expected. This change - introduces some backwards breaks in the API: Some config classes - (Config, SolrConfig, IndexSchema) were changed to take - org.xml.sax.InputSource instead of InputStream. There may also be some - backwards breaks in existing config files, it is recommended to check - your config files / XSLTs and replace all XIncludes/HREFs that were + are fixed to be resolved using the URI standard (RFC 2396). The system + identifier is no longer a plain filename with path, it gets initialized + using a custom URI scheme "solrres:". This scheme is resolved using a + EntityResolver that utilizes ResourceLoader + (org.apache.solr.common.util.SystemIdResolver). This makes all relative + pathes in Solr's config files behave like expected. This change + introduces some backwards breaks in the API: Some config classes + (Config, SolrConfig, IndexSchema) were changed to take + org.xml.sax.InputSource instead of InputStream. There may also be some + backwards breaks in existing config files, it is recommended to check + your config files / XSLTs and replace all XIncludes/HREFs that were hacked to use absolute paths to use relative ones. (uschindler) +* SOLR-309: Fix FieldType so setting an analyzer on a FieldType that + doesn't expect it will generate an error. Practically speaking this + means that Solr will now correctly generate an error on + initialization if the schema.xml contains an analyzer configuration + for a fieldType that does not use TextField. (hossman) + +* SOLR-2192: StreamingUpdateSolrServer.blockUntilFinished was not + thread safe and could throw an exception. (yonik) Other Changes ---------------------- From d112c045e626d5ed02b852905683f3d191187aef Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Wed, 30 Mar 2011 21:34:43 +0000 Subject: [PATCH 040/200] alias for javadocs/javadoc git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087087 13f79535-47bb-0310-9956-ffa450edef68 --- solr/build.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/build.xml b/solr/build.xml index 3c3c6090556..9370217be5c 100644 --- a/solr/build.xml +++ b/solr/build.xml @@ -222,6 +222,7 @@ + From 2fc69540a9353e837e4e2cc923ead514140921f5 Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Wed, 30 Mar 2011 21:35:11 +0000 Subject: [PATCH 041/200] alias for javadocs/javadoc git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087088 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/build.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/lucene/build.xml b/lucene/build.xml index a35460e9be2..cd471a9f3ce 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -152,6 +152,7 @@ DEPRECATED - Doing Nothing. See http://wiki.apache.org/lucene-java/HowToUpdateTheWebsite + From bbed5715cb97c2a476a61819dbee9e60f8edd313 Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Wed, 30 Mar 2011 22:29:35 +0000 Subject: [PATCH 042/200] Removed obsolete CVS version control config files git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087112 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/.cvsignore | 5 ----- lucene/contrib/highlighter/.cvsignore | 2 -- 2 files changed, 7 deletions(-) delete mode 100644 lucene/.cvsignore delete mode 100644 lucene/contrib/highlighter/.cvsignore diff --git a/lucene/.cvsignore b/lucene/.cvsignore deleted file mode 100644 index bd8ad3a3a28..00000000000 --- a/lucene/.cvsignore +++ /dev/null @@ -1,5 +0,0 @@ -build -dist -*~ -velocity.log -build.properties diff --git a/lucene/contrib/highlighter/.cvsignore b/lucene/contrib/highlighter/.cvsignore deleted file mode 100644 index 9d0b71a3c79..00000000000 --- a/lucene/contrib/highlighter/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -build -dist From 82f0640b490d82c89d5de8f249d58441083c02b0 Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Thu, 31 Mar 2011 14:29:09 +0000 Subject: [PATCH 043/200] update site for 3.1 git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087314 13f79535-47bb-0310-9956-ffa450edef68 --- solr/site/features.html | 22 ++--- solr/site/features.pdf | Bin 14676 -> 20288 bytes solr/site/index.html | 89 ++++++++++++++------ solr/site/index.pdf | Bin 73739 -> 118517 bytes solr/site/issue_tracking.html | 4 +- solr/site/issue_tracking.pdf | Bin 2304 -> 6466 bytes solr/site/linkmap.html | 4 +- solr/site/linkmap.pdf | Bin 2442 -> 9826 bytes solr/site/mailing_lists.html | 10 +-- solr/site/mailing_lists.pdf | Bin 9000 -> 13709 bytes solr/site/skin/images/apache-thanks.png | Bin 0 -> 4840 bytes solr/site/skin/images/built-with-cocoon.gif | Bin 0 -> 2252 bytes solr/site/tutorial.html | 86 +++++++++---------- solr/site/tutorial.pdf | Bin 47122 -> 52104 bytes solr/site/version_control.html | 12 +-- solr/site/version_control.pdf | Bin 8430 -> 13646 bytes 16 files changed, 133 insertions(+), 94 deletions(-) create mode 100644 solr/site/skin/images/apache-thanks.png create mode 100644 solr/site/skin/images/built-with-cocoon.gif diff --git a/solr/site/features.html b/solr/site/features.html index 6874449de23..cdd83f7685f 100755 --- a/solr/site/features.html +++ b/solr/site/features.html @@ -3,7 +3,7 @@ - + Introduction to The Solr Enterprise Search Server @@ -199,7 +199,7 @@ document.write("Last Published: " + document.lastModified); |alternative credits +-->

-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
BENju<+X^!+pvIEgRTUYbNT2N(x!pLZq@$d$4w-kv-v6*~lqT^gE%Rk+m{x1-hpx?r;Z3ed^}Q z;#WtQf~2qM)C^y&g>=eZRQ(aJebvs2?mK&&(4^)A1-x>4ZU zYiiSk(5BdFTPV_TE7~qr3v0*Kdh=TA0;yV1(bI61!(uQVY0~XOadRd^S+~kyu2PL{ z2lly1O5Enro||$cohAQn8_)3Du0^s!DDpXK3%Sah^Efu+Rn{`;(d~q5CC@TksT(cj z#)$$J6sCA()dgpxbFdWL4=X9B(_5iRt4X-;>w(8#BgJ-GZU1O+v}RnKn3T)&29$wG zOTKV!Q|)ina<7vse32N21c>5WAjz=@hY|t+SRZ+#ND;;ZeZ zy|k6{7Z6H`45B^oMn8I}wzP;qso`-dhHq_6dq0#6x_H5TT!gp7t}Rt_$zVTR1c+e- zT$@T$saE5`jWV9J#p(6Z&TdMqIL>GMiqGdfxUlJ_<*IwstfP6#_=kJgO%5m5X~pBI z&Bf)k=E|1EQzzjMkNUT4xEg20WhFV{L~&+;inuV_yJM z3nN}j_MG(!7bqL$TT8`0?S_hL1tmV3T)puHZz0hrw*jYa+p%`uY*KrMXGlSk3iW|I z6iwSld}2hot2dfi^%*a{3-*%%MPdetK|!3TV0;>=bIQ|b);#0=t{sZ4ch1hf>$lv5 zWiMC?*yv0GShVMGqt2P7vzwir?m!pac^!d#bT^gRPL%Sl6UrwTp7IolRb_kpXmF1$a8|=YX zClyzMIQysKe4R{r?Br^^Z(70O2CvQ%UVby#Byyf?w!uxi-oOsZ!dT{uNVA<3BPX(L ziKR(#((Od(q*dI|OVU<49kGIh4(aOp8l>_f#Yo=GaEkag5pnI$tSXjUqR}SZO3AV8 zv)q({a5!?>m#muw%7vGF2PP|OpcMN}uJ0`OmEf(jnJ1fV`4kLA{Fc&nTygSgB)!bp z2Mk(~n+<2R7>GN{))-pw`MsRWwt(D^Em168IA)Qcx)G10RMG{(KjL$VjFsO7M08xL zctKEmv@>`)WblFNV|JX_&y)Dc)~Bqtm^GXcyXty2-+{+0iO!f{W40J^#Yo7H5xeFOavS~VQA!3QyGQDEgTB(*G z#L=TjO9XP&((W`DN*_75b0HqCl=xdRRLqPc@J0Ff5&<3=&MYl7hA5$x_ zVD%iY5XrRchX)QW>~#WY7{E!L!ZUF6=DT z{m=!9wpKx)?m=;oJjc3;RyDGT)u#L5wB~c$%NbF2P@I*minX#-^VE43trQ4lU!N%H zqcU+^gX#vG)h2@1-#aZ9lN+}I3s@*qlfceboIgUNKVR9B4G1?INtXhSCk zlV&1(RuYu+kWMfiqCJaBJ@B4xbQTp!MpCRKMz~$@Os`_FOz{wCwI`Pv4Ciq>^1Ky? z(rL{+SE-+d)^N?gmplGd=*C6@p4BM0AP#|xH#-`PYH2i`z0xd^_Ldj>Mk|20XTI^3 z#j6?YP{}ozkvp2`kE6%dP{k_2yqF~tR~6jd`vOMTi{WV(x#87XIGXWYWl2OXqf@ZR zrxRy;pR!EK*AUEj!9K?#MJ1C}qwO}zU0hBtC3e~Dmd>$Z8(JAf-4QTer*apy+f;0* zH)(V48fWm)7l&8BEi&+nD=^sZ3YGGuKqR_@CV!&4YoU)MBQE!Pxw|C_rht=LHZH>F)b*YC1iXz<{?)mLN^lUz|9zsr( zXVc4{vL5Q2g*WRVYgh>1tcQAowwoLO06U0o4a88xa0vCd@7IRu5QCkxW-Y`}!+3~6 zo*Cm>+mLAOhG`N*jZU!F8sqnR*cf=0y|XqN9e!ggi7`g39Sz>C5GV*al5j177z7Uf zyKaW3<9`G-a1oDSH~B&!q(Q)M;H9~jFt95&@FKvS5QP}se;+R%3^x21QTncb(0!;V zC|eHh$TS#YVAo7R>%Z$M`RWuBR-ptKuWrFW^NQft97ZtonlcO&fRG-!gEKs`Z^6!{%KDI?c(J+OFfa6~b$2Sd7T4Fv*V=Yv3 zH>SW_T)QG!&ENlbVE@S8{)KW77{q2j&sLwi3PLtMJ=$JNR0#$%@P-8KMgr}BPiucj z^W=e-|NbBFpz0j=!a@4^D<{dn^s6!^u5Sp2o=-sYSk>~}X`Y?HRA3N<(+?;+_Xk}& zjnUvW)GpGm6RsH(?>F$!kMCZ68!-(DEt$X{_H8sH!D*=k^n3!&zj!&8<`0m1s-e%Z zZJ1+y?F@yYpUaNEwD<<5805jzlBMEWN=R+QH1jPam&D=x?oT>$evYi5E}?=TlqLCn zyuW!!_Y-f3G%;M{K^9>m*2>=hNf-G~@fKfQ9Y4q1dDgf==L2o2^>SH5%Ku;S!8eyB zI>+Nsj7A%D4334MP7Q={{8`8IeuRJLqWY8AMZN||M49lqtEf+z|-2D zl_32ktbNkKCre8~z5eafI#~;A3C_jg#`Opv5qu~0csgDr?M1le;Zw5gQveWN6?`id zJZ<4w$hpbOL7y>B#^)pfpo9MfMf0NW`uM=G4G+^afWM!c=%AX zwji&$vx1x*Re^AaQq~Nk;)H)D#|Wjq$@6&KRiMvT3@QkZ@>^&A;22v&tmmk#;zO|8 zjB|7|Pj>y4c5QU9itV|5-7c31by&V$*QY!eE{03&es_#-t$5yfmRsI-9cn*^4z^DA zm{~K^HPp(?20h6w>AJ~>_Svs zEiGgLSyp#^c2CQxTK>kyWp5c9MiVE8cVf$O1lRX&Tg$!eh)B+;qbp~6N*BC*LRTz1 z*f3FuNw$UgDZNatsk$Frb3)9iE7|4)4wB0Z=C#1QScNkh1Q-kJX?+T zd1ezCW2n1*X2m78qUz~7!e&Rg24+p;(rW~#*k5KYBE_1duUQF5>Bxw=UTd|NTz(E!(7JZoOv3(k$TM|WG2m= z9ZT%QmX%&+i1JOZB{v(D8^c8s5_B!eq$LR6ylzAal&^ufu9mtjd54dRi5ldMdJlCL?=_+v9WBDBW@8yH^3Hg3`?+FM8< z=Hw#Va$r4(x!pygo8v2!T42>4>}|POX;B%UgYphj!gwIfAQQ)BkoK|td>ZnT_{e41 zs~}vn>hJ6%j^x}*Hqk(%v6G9=21cQNZOt|Op-Y4dF5~L4ZXxm5O$v^7F0hBlnCSOG zX%8Wt8c3AW}AZP2LsJI9FS^$bECHWGfyWAkN2}gaml7uyj$r)=OIH2yO!`Y@&P->vM%PfyF6q+0$S9c?G3ryO~ zA}&ORn?96yyaZUkpDRjSlUA0QYn(YLe%k;vn-1u;4 zow2QV3^g|cL~ZX#lJojT!P>J}GG;Gtz4dX4^~8hQ{WjBS2g7@1LVJr%D%EtwG8UDa zVP+ee%Hk&z9<%sv=@%E|^S3Ei~y8bOX>hKcOW&Stb|D^@VH!J0!im|=Ir8AOE{ z*LTAiemBJ7cS9VMi`MVsMlV0z^^6aFV;QV(RABi>Yssx)@eAfO*#6R5ZtL%DAr)Sd z!L%}$r3++&{$&|;Ovm&afRg0hi~`{|+HM{4w88$0g5byB^DX;u&zJ$=RcU#HwM`q= zTPyJws~5AC_S1bY6oS%yT_}ii1dkWkf#Be8Nr4BM?e>gZy=yqth9*IzO|Lz17N@k;DF?Txu#N5h!jAJKT5ay~{w>-5eF zIxKf^A7Ej0M)?P1P~$-Zjr;G`&hi9avuvrDxSvTzBGOui?>#v2muAXPqYrZYeiR->;wNh) z;jB;BqK`1VO)nqA@J2K2K`wRmf>B0`&JjlckL@3E$0J-nRk}gMpJ3jJ_^R*$YLloE zZyTczq1IK1!5*kF)#z+M>a=@QoUhvNjhh}Gc~q9hs}vY_KbVv52)qva9f4Qn4+xmW zP`mjFf9uS9=kH%Y^ai>vpBVNTh_6JwgZQf00YtM1YBxUtQP&T&#sPOY9~6V3!gU6? zkHVuE8o%|&4I!lsp4Qvxk0;buIegXUcb)ipc^`pSlHbvLRgv#>;+@YPWkCDNmoR-6 zM)VtK{Up?*HYE)S)<@2eox21*e)!!&2+q5Vg-{YUX!_4bOnx={YM*^n>;~a=Wpba0 zjTU`yjJ43?;23I$qCaD?2l&i#t8tk@X;>%zeG(u709L_JPwlq6>X%w0_5-QGc^lj# zejq9SB52RNAO%7G3*uK+d?0m{*FTW#VRc-?F6IU4ONZp)BK>T5Fk+C}B>IgWsmC5W zk|JCJts?{(eYHFDR_Rj&0o4x>{Ay=!^T(H{iM{I$ghG|TN$5Moe48x3)FJzb-MzoB z;`XNDztk1}*wq`S)R!_;+!4Oo>6=#mQm4#grv|fpgPJDB)HZ{cANLg;(5@`4FE3+V zo*-;flXdVxF#gfCHRRvZuk<{VEeP4uuXNYE7VI#LN2f1u(qQJ7G`LgA$L$Z;XF=aD zuZzg~4LlYPTEp8U4G&b0YmmzBMR>jcK82_Fh{`3)qobsQQ*I#FKVG3 zgWr00@SBC-x-Qe&wHpRrNDd4LcZ`u7oPLtJLV-SC0ATgFY75twG0exA+zU8S0)S~4 zHf?{XA^q~>z9D?mIeOj3{IPTVfn;9BcakE5;phd5&!Lcdl7+tyH(8!OL@VoCwfpRV zv|o^aexG6qJy_E8g*Vdt0s`<>(hs{$UwC88_g{FW&KnNkVn~~Dc$ned;h@*qZN0qF zfM9?Ir1=B?M8rD_zI}iOlS#<*zP|YVh=`{5aa^kv%b36LPei`YP@t9PWbm2EoUSq%X0)^ga&BthAWqX9ym5MelGhL7=6>(t`d> zU-%~~-nGUX4kVbsK6wA17}4}T76Rv-52$Eplsg$f-i^KPPfN*tU1zM^-c~mMRfWwD z&Hau3f&Tmh0!tCp%jU&sgSseMb^z_q%i(7mtI6;CHW*;gjqlpvLOTF?Zqw4}KDDu0 z>cIDH;5KY-!?h!mch5m>QLRk$yEYUfw5rhW+DHP|?8fifP>j^Z_+1-3r)64r*9MXO zn#O+D1~vG!vd!<>P{@5}?w2&di~=K>`z5usg73cvPIBgcDNSKNJqOOGPi??yrZGan zEA<&Scuq@O{PaC2;QG`?LW%=3e3WV2NXQju{vM;*Oy2bY&zboE%9Wb&Wwj~g-Mc8x zYDq2LwZTb|=CA&)4Fz{Y;Q9A$6#YRPW8xQ-Vl0PrdU{G}L89p5HDn5@n{(+7F4QA4(IQ)600YpTY zOlR==H1RkWP4Yo}ptFe&WSaOurU{Vg zXM71J34Z*Rq={edo=Yg8k%5%N{9t$&wsST%|`vsAKYH!c)lG+~i>3a~m zVQPaElBPEETnn1cy!Jr}6K_Cabkq6@{NlB(&^uUwT}*Qd2_a<9_>yqW_Y5;&X5v#= zU72CQKro;C{Xln;kj29UA1F@~ouSmB`8m?0H%I{4{M - + Solr Version Control System @@ -199,7 +199,7 @@ document.write("Last Published: " + document.lastModified); |alternative credits +-->
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
From fdf66a15b1da07ec836fa491660c840595ef3d15 Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Thu, 31 Mar 2011 19:25:01 +0000 Subject: [PATCH 045/200] Fixed incorrect specification of testResource directory src/test-files/; added javax.servlet:servlet-api as a test dependency. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087414 13f79535-47bb-0310-9956-ffa450edef68 --- .../maven/solr/contrib/analysis-extras/pom.xml.template | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template b/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template index ee6c8392384..4ecff9241ff 100644 --- a/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template +++ b/dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template @@ -85,6 +85,11 @@ junit test + + javax.servlet + servlet-api + test + ${build-directory} @@ -94,7 +99,7 @@ src/test - test-files + src/test-files ../../src/test-files From 937ee38c7ff6a51344ad59902b1e66e384d04524 Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Thu, 31 Mar 2011 19:56:47 +0000 Subject: [PATCH 046/200] LUCENE-3006: fail on javadoc warnings git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087426 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 5 +++++ lucene/build.xml | 1 + lucene/common-build.xml | 26 +++++++++++++++++++++++++- solr/CHANGES.txt | 2 ++ solr/common-build.xml | 20 ++++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 0ec3f79127e..8ea03b6dcc2 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -379,6 +379,11 @@ Test Cases stop iterating if at least 'tests.iter.min' ran and a failure occured. (Shai Erera, Chris Hostetter) +Build + +* LUCENE-3006: Building javadocs will fail on warnings by default. Override with -Dfailonjavadocwarning=false (sarowe, gsingers) + + ======================= Lucene 3.x (not yet released) ======================= Changes in backwards compatibility policy diff --git a/lucene/build.xml b/lucene/build.xml index 1d12084cf76..3a0a522249a 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -200,6 +200,7 @@ + diff --git a/lucene/common-build.xml b/lucene/common-build.xml index 7415bbcd916..3225f7e0774 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -103,6 +103,7 @@ + @@ -762,7 +763,8 @@ - + + @@ -772,6 +774,7 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 0f36491e0be..df4a7231fb1 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -211,6 +211,8 @@ Documentation * SOLR-2232: Improved README info on solr.solr.home in examples (Eric Pugh and hossman) +* LUCENE-3006: Building javadocs will fail on warnings by default. Override with -Dfailonjavadocwarning=false (sarowe, gsingers) + ================== 3.2.0-dev ================== Versions of Major Components --------------------- diff --git a/solr/common-build.xml b/solr/common-build.xml index 420f75726e4..861ff237062 100644 --- a/solr/common-build.xml +++ b/solr/common-build.xml @@ -330,6 +330,7 @@ + @@ -339,6 +340,7 @@ + + + + + + + + + + + + + + + + + + + From 004524dbecd2c15909c54a7deeb77d30c92f5688 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 31 Mar 2011 19:59:15 +0000 Subject: [PATCH 047/200] added synchronization to prevent threads from seeing inconsistent name / number mappings when asserting for consistency git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087427 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/src/java/org/apache/lucene/index/FieldInfos.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index 5540f5f507c..e8ed8ae127e 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -187,7 +187,7 @@ public final class FieldInfos implements Iterable { } // used by assert - boolean containsConsistent(Integer number, String name) { + synchronized boolean containsConsistent(Integer number, String name) { return name.equals(numberToName.get(number)) && number.equals(nameToNumber.get(name)); } From 74c9b8887f210186a30a1750f0d012c17a77d234 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 31 Mar 2011 20:13:22 +0000 Subject: [PATCH 048/200] SOLR-2338: improved per-field similarity integration into schema.xml git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087430 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 + .../apache/solr/common/util/NamedList.java | 18 ++- .../org/apache/solr/schema/FieldType.java | 29 +++++ .../org/apache/solr/schema/IndexSchema.java | 90 ++++++++++----- .../apache/solr/schema/SimilarityFactory.java | 4 +- .../schema/SimilarityProviderFactory.java | 34 ++++++ .../solr/search/SolrSimilarityProvider.java | 56 +++++++++ solr/src/test-files/solr/conf/schema.xml | 47 +++++++- .../solr/schema/CustomSimilarityFactory.java | 4 +- .../CustomSimilarityProviderFactory.java | 34 ++++++ .../apache/solr/schema/IndexSchemaTest.java | 8 +- .../schema/MockConfigurableSimilarity.java | 4 +- .../MockConfigurableSimilarityProvider.java | 32 ++++++ .../solr/schema/TestPerFieldSimilarity.java | 106 ++++++++++++++++++ 14 files changed, 424 insertions(+), 45 deletions(-) create mode 100644 solr/src/java/org/apache/solr/schema/SimilarityProviderFactory.java create mode 100644 solr/src/java/org/apache/solr/search/SolrSimilarityProvider.java create mode 100644 solr/src/test/org/apache/solr/schema/CustomSimilarityProviderFactory.java create mode 100644 solr/src/test/org/apache/solr/schema/MockConfigurableSimilarityProvider.java create mode 100644 solr/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index df4a7231fb1..f341eaf76f1 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -117,6 +117,9 @@ New Features * SOLR-2396: Add CollationField, which is much more efficient than the Solr 3.x CollationKeyFilterFactory, and also supports Locale-sensitive range queries. (rmuir) + +* SOLR-2338: Add support for using in a schema's fieldType, + for customizing scoring on a per-field basis. (hossman, yonik, rmuir) Optimizations diff --git a/solr/src/common/org/apache/solr/common/util/NamedList.java b/solr/src/common/org/apache/solr/common/util/NamedList.java index 579a7f29008..10b1996e608 100644 --- a/solr/src/common/org/apache/solr/common/util/NamedList.java +++ b/solr/src/common/org/apache/solr/common/util/NamedList.java @@ -43,7 +43,7 @@ import java.io.Serializable; * by key, so ResponseWriters that output to a format such as JSON will normally * choose a data structure that allows order to be easily preserved in various * clients (i.e. not a straight map). - * If access by key is more important, see {@link SimpleOrderedMap}, + * If access by key is more important for serialization, see {@link SimpleOrderedMap}, * or simply use a regular {@link Map} *

* @@ -200,10 +200,14 @@ public class NamedList implements Cloneable, Serializable, Iterable + * NOTE: this runs in linear time (it scans starting at the + * beginning of the list until it finds the first pair with + * the specified name). * @return null if not found or if the value stored was null. * @see #indexOf * @see #get(String,int) + * */ public T get(String name) { return get(name,0); @@ -212,7 +216,10 @@ public class NamedList implements Cloneable, Serializable, Iterable + * NOTE: this runs in linear time (it scans starting at the + * specified position until it finds the first pair with + * the specified name). * @return null if not found or if the value stored was null. * @see #indexOf */ @@ -377,6 +384,11 @@ public class NamedList implements Cloneable, Serializable, Iterable + * The default implementation returns null, which means this type + * has no custom similarity associated with it. + *

+ * + * This method exists to internally support SolrSimilarityProvider. + * Custom application code interested in a field's Similarity should + * instead query via the searcher's SimilarityProvider. + * @lucene.internal + */ + public Similarity getSimilarity() { + return similarity; + } + + /** + * Sets the Similarity used when scoring fields of this type + * @lucene.internal + */ + public void setSimilarity(Similarity similarity) { + this.similarity = similarity; + } + /** * calls back to TextResponseWriter to write the field value */ diff --git a/solr/src/java/org/apache/solr/schema/IndexSchema.java b/solr/src/java/org/apache/solr/schema/IndexSchema.java index e7732e3d3d7..b6b7b6768f5 100644 --- a/solr/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/src/java/org/apache/solr/schema/IndexSchema.java @@ -20,7 +20,9 @@ package org.apache.solr.schema; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Similarity; import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.util.Version; @@ -28,6 +30,7 @@ import org.apache.solr.common.ResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.DOMUtil; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SystemIdResolver; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.Config; @@ -37,6 +40,7 @@ import org.apache.solr.analysis.TokenFilterFactory; import org.apache.solr.analysis.TokenizerChain; import org.apache.solr.analysis.TokenizerFactory; import org.apache.solr.search.SolrQueryParser; +import org.apache.solr.search.SolrSimilarityProvider; import org.apache.solr.util.plugin.AbstractPluginLoader; import org.apache.solr.util.plugin.SolrCoreAware; import org.w3c.dom.*; @@ -185,19 +189,22 @@ public final class IndexSchema { */ public Collection getRequiredFields() { return requiredFields; } - private SimilarityFactory similarityFactory; + private SimilarityProviderFactory similarityProviderFactory; /** - * Returns the Similarity used for this index + * Returns the SimilarityProvider used for this index */ - public SimilarityProvider getSimilarityProvider() { return similarityFactory.getSimilarityProvider(); } + public SimilarityProvider getSimilarityProvider() { return similarityProviderFactory.getSimilarityProvider(this); } /** - * Returns the SimilarityFactory used for this index + * Returns the SimilarityProviderFactory used for this index */ - public SimilarityFactory getSimilarityFactory() { return similarityFactory; } - + public SimilarityProviderFactory getSimilarityProviderFactory() { return similarityProviderFactory; } + private Similarity fallbackSimilarity; + + /** fallback similarity, in the case a field doesnt specify */ + public Similarity getFallbackSimilarity() { return fallbackSimilarity; } /** * Returns the Analyzer used when indexing documents for this index @@ -387,6 +394,11 @@ public final class IndexSchema { expression = "./analyzer[not(@type)] | ./analyzer[@type='index']"; anode = (Node)xpath.evaluate(expression, node, XPathConstants.NODE); Analyzer analyzer = readAnalyzer(anode); + + // a custom similarity[Factory] + expression = "./similarity"; + anode = (Node)xpath.evaluate(expression, node, XPathConstants.NODE); + Similarity similarity = readSimilarity(anode); if (queryAnalyzer==null) queryAnalyzer=analyzer; if (analyzer==null) analyzer=queryAnalyzer; @@ -394,6 +406,9 @@ public final class IndexSchema { ft.setAnalyzer(analyzer); ft.setQueryAnalyzer(queryAnalyzer); } + if (similarity!=null) { + ft.setSimilarity(similarity); + } if (ft instanceof SchemaAware){ schemaAware.add((SchemaAware) ft); } @@ -491,36 +506,31 @@ public final class IndexSchema { // stuff it in a normal array for faster access dynamicFields = dFields.toArray(new DynamicField[dFields.size()]); - Node node = (Node) xpath.evaluate("/schema/similarity", document, XPathConstants.NODE); + Similarity similarity = readSimilarity(node); + fallbackSimilarity = similarity == null ? new DefaultSimilarity() : similarity; + + node = (Node) xpath.evaluate("/schema/similarityProvider", document, XPathConstants.NODE); if (node==null) { - similarityFactory = new SimilarityFactory() { + final SolrSimilarityProvider provider = new SolrSimilarityProvider(this); + similarityProviderFactory = new SimilarityProviderFactory() { @Override - public SimilarityProvider getSimilarityProvider() { - return IndexSearcher.getDefaultSimilarityProvider(); + public SolrSimilarityProvider getSimilarityProvider(IndexSchema schema) { + return provider; } }; - log.debug("using default similarity"); + log.debug("using default similarityProvider"); } else { final Object obj = loader.newInstance(((Element) node).getAttribute("class")); - if (obj instanceof SimilarityFactory) { - // configure a factory, get a similarity back - SolrParams params = SolrParams.toSolrParams(DOMUtil.childNodesToNamedList(node)); - similarityFactory = (SimilarityFactory)obj; - similarityFactory.init(params); - } else { - // just like always, assume it's a SimilarityProvider and get a ClassCastException - reasonable error handling - similarityFactory = new SimilarityFactory() { - @Override - public SimilarityProvider getSimilarityProvider() { - return (SimilarityProvider) obj; - } - }; + // just like always, assume it's a SimilarityProviderFactory and get a ClassCastException - reasonable error handling + // configure a factory, get a similarity back + NamedList args = DOMUtil.childNodesToNamedList(node); + similarityProviderFactory = (SimilarityProviderFactory)obj; + similarityProviderFactory.init(args); + if (similarityProviderFactory instanceof SchemaAware){ + schemaAware.add((SchemaAware) similarityProviderFactory); } - if (similarityFactory instanceof SchemaAware){ - schemaAware.add((SchemaAware) similarityFactory); - } - log.debug("using similarity factory" + similarityFactory.getClass().getName()); + log.debug("using similarityProvider factory" + similarityProviderFactory.getClass().getName()); } node = (Node) xpath.evaluate("/schema/defaultSearchField/text()", document, XPathConstants.NODE); @@ -750,6 +760,30 @@ public final class IndexSchema { return newArr; } + private Similarity readSimilarity(Node node) throws XPathExpressionException { + if (node==null) { + return null; + } else { + SimilarityFactory similarityFactory; + final Object obj = loader.newInstance(((Element) node).getAttribute("class")); + if (obj instanceof SimilarityFactory) { + // configure a factory, get a similarity back + SolrParams params = SolrParams.toSolrParams(DOMUtil.childNodesToNamedList(node)); + similarityFactory = (SimilarityFactory)obj; + similarityFactory.init(params); + } else { + // just like always, assume it's a Similarity and get a ClassCastException - reasonable error handling + similarityFactory = new SimilarityFactory() { + @Override + public Similarity getSimilarity() { + return (Similarity) obj; + } + }; + } + return similarityFactory.getSimilarity(); + } + } + // // // diff --git a/solr/src/java/org/apache/solr/schema/SimilarityFactory.java b/solr/src/java/org/apache/solr/schema/SimilarityFactory.java index f2e29d575b5..25b03a0ad68 100644 --- a/solr/src/java/org/apache/solr/schema/SimilarityFactory.java +++ b/solr/src/java/org/apache/solr/schema/SimilarityFactory.java @@ -16,7 +16,7 @@ package org.apache.solr.schema; * limitations under the License. */ -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.Similarity; import org.apache.solr.common.params.SolrParams; public abstract class SimilarityFactory { @@ -25,5 +25,5 @@ public abstract class SimilarityFactory { public void init(SolrParams params) { this.params = params; } public SolrParams getParams() { return params; } - public abstract SimilarityProvider getSimilarityProvider(); + public abstract Similarity getSimilarity(); } diff --git a/solr/src/java/org/apache/solr/schema/SimilarityProviderFactory.java b/solr/src/java/org/apache/solr/schema/SimilarityProviderFactory.java new file mode 100644 index 00000000000..dafb9411df2 --- /dev/null +++ b/solr/src/java/org/apache/solr/schema/SimilarityProviderFactory.java @@ -0,0 +1,34 @@ +package org.apache.solr.schema; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.solr.common.util.NamedList; +import org.apache.solr.search.SolrSimilarityProvider; + +/** + * Expert: Factory to provide a {@link SolrSimilarityProvider}. + *

+ * Usually you would implement this if you want to customize the + * scoring routines that are not field-specific, such as coord() and queryNorm(). + * Most scoring customization happens in the fieldtype's Similarity + */ +public abstract class SimilarityProviderFactory { + public void init(NamedList args) {} + + public abstract SolrSimilarityProvider getSimilarityProvider(IndexSchema schema); +} diff --git a/solr/src/java/org/apache/solr/search/SolrSimilarityProvider.java b/solr/src/java/org/apache/solr/search/SolrSimilarityProvider.java new file mode 100644 index 00000000000..2fe437fd999 --- /dev/null +++ b/solr/src/java/org/apache/solr/search/SolrSimilarityProvider.java @@ -0,0 +1,56 @@ +package org.apache.solr.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.DefaultSimilarityProvider; +import org.apache.lucene.search.Similarity; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; + +/** + * SimilarityProvider that uses the default Lucene implementation, unless + * otherwise specified by the fieldtype. + *

+ * You can extend this class to customize the behavior of the parts + * of lucene's ranking system that are not field-specific, such as + * {@link #coord(int, int)} and {@link #queryNorm(float)}. + */ +public class SolrSimilarityProvider extends DefaultSimilarityProvider { + private final IndexSchema schema; + + public SolrSimilarityProvider(IndexSchema schema) { + this.schema = schema; + } + + /** + * Solr implementation delegates to the fieldtype's similarity. + * If this does not exist, uses the schema's default similarity. + */ + // note: this is intentionally final, to maintain consistency with + // whatever is specified in the the schema! + @Override + public final Similarity get(String field) { + FieldType fieldType = schema.getFieldTypeNoEx(field); + if (fieldType == null) { + return schema.getFallbackSimilarity(); + } else { + Similarity similarity = fieldType.getSimilarity(); + return similarity == null ? schema.getFallbackSimilarity() : similarity; + } + } +} diff --git a/solr/src/test-files/solr/conf/schema.xml b/solr/src/test-files/solr/conf/schema.xml index 6dabdde653b..f0fa272ff2b 100644 --- a/solr/src/test-files/solr/conf/schema.xml +++ b/solr/src/test-files/solr/conf/schema.xml @@ -391,6 +391,32 @@ + + + + + + + + + + + + + + + + + is there an echo? + + + + + + + + + @@ -496,6 +522,9 @@ + + + @@ -574,6 +603,10 @@ + + + + text @@ -607,12 +640,18 @@ - - + is there an echo? - + + + + I am your default sim + diff --git a/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java b/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java index 03fbaecaec2..98267cce285 100644 --- a/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java +++ b/solr/src/test/org/apache/solr/schema/CustomSimilarityFactory.java @@ -16,11 +16,11 @@ */ package org.apache.solr.schema; -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.Similarity; public class CustomSimilarityFactory extends SimilarityFactory { @Override - public SimilarityProvider getSimilarityProvider() { + public Similarity getSimilarity() { return new MockConfigurableSimilarity(params.get("echo")); } } diff --git a/solr/src/test/org/apache/solr/schema/CustomSimilarityProviderFactory.java b/solr/src/test/org/apache/solr/schema/CustomSimilarityProviderFactory.java new file mode 100644 index 00000000000..68de144e6b7 --- /dev/null +++ b/solr/src/test/org/apache/solr/schema/CustomSimilarityProviderFactory.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import org.apache.solr.common.util.NamedList; +import org.apache.solr.search.SolrSimilarityProvider; + +public class CustomSimilarityProviderFactory extends SimilarityProviderFactory { + String echoParam; + + @Override + public void init(NamedList args) { + echoParam = (String) args.get("echo"); + } + + @Override + public SolrSimilarityProvider getSimilarityProvider(IndexSchema schema) { + return new MockConfigurableSimilarityProvider(schema, echoParam); + } +} diff --git a/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java b/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java index cb176d8dee8..737cb90ffc1 100644 --- a/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java +++ b/solr/src/test/org/apache/solr/schema/IndexSchemaTest.java @@ -81,11 +81,11 @@ public class IndexSchemaTest extends SolrTestCaseJ4 { } @Test - public void testSimilarityFactory() { + public void testSimilarityProviderFactory() { SolrCore core = h.getCore(); - SimilarityProvider similarity = core.getSchema().getSimilarityProvider(); - assertTrue("wrong class", similarity instanceof MockConfigurableSimilarity); - assertEquals("is there an echo?", ((MockConfigurableSimilarity)similarity).getPassthrough()); + SimilarityProvider similarityProvider = core.getSchema().getSimilarityProvider(); + assertTrue("wrong class", similarityProvider instanceof MockConfigurableSimilarityProvider); + assertEquals("is there an echo?", ((MockConfigurableSimilarityProvider)similarityProvider).getPassthrough()); } @Test diff --git a/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java b/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java index 6b8f6c1d362..36021fab4e7 100644 --- a/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java +++ b/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java @@ -16,9 +16,9 @@ */ package org.apache.solr.schema; -import org.apache.lucene.search.DefaultSimilarityProvider; +import org.apache.lucene.search.DefaultSimilarity; -public class MockConfigurableSimilarity extends DefaultSimilarityProvider { +public class MockConfigurableSimilarity extends DefaultSimilarity { private String passthrough; public MockConfigurableSimilarity(String passthrough) { diff --git a/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarityProvider.java b/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarityProvider.java new file mode 100644 index 00000000000..3a6e8a40ed0 --- /dev/null +++ b/solr/src/test/org/apache/solr/schema/MockConfigurableSimilarityProvider.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import org.apache.solr.search.SolrSimilarityProvider; + +public class MockConfigurableSimilarityProvider extends SolrSimilarityProvider { + private String passthrough; + + public MockConfigurableSimilarityProvider(IndexSchema schema, String passthrough) { + super(schema); + this.passthrough = passthrough; + } + + public String getPassthrough() { + return passthrough; + } +} diff --git a/solr/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java b/solr/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java new file mode 100644 index 00000000000..3aa81c30c84 --- /dev/null +++ b/solr/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java @@ -0,0 +1,106 @@ +package org.apache.solr.schema; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.misc.SweetSpotSimilarity; +import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.Similarity; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.BeforeClass; + +/** + * Tests per-field similarity support in the schema + */ +public class TestPerFieldSimilarity extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml","schema.xml"); + } + + /** test a field where the sim is specified directly */ + public void testDirect() throws Exception { + SolrCore core = h.getCore(); + RefCounted searcher = core.getSearcher(); + Similarity sim = searcher.get().getSimilarityProvider().get("sim1text"); + assertEquals(SweetSpotSimilarity.class, sim.getClass()); + searcher.decref(); + } + + /** ... and for a dynamic field */ + public void testDirectDynamic() throws Exception { + SolrCore core = h.getCore(); + RefCounted searcher = core.getSearcher(); + Similarity sim = searcher.get().getSimilarityProvider().get("text_sim1"); + assertEquals(SweetSpotSimilarity.class, sim.getClass()); + searcher.decref(); + } + + /** test a field where a configurable sim factory is defined */ + public void testFactory() throws Exception { + SolrCore core = h.getCore(); + RefCounted searcher = core.getSearcher(); + Similarity sim = searcher.get().getSimilarityProvider().get("sim2text"); + assertEquals(MockConfigurableSimilarity.class, sim.getClass()); + assertEquals("is there an echo?", ((MockConfigurableSimilarity)sim).getPassthrough()); + searcher.decref(); + } + + /** ... and for a dynamic field */ + public void testFactoryDynamic() throws Exception { + SolrCore core = h.getCore(); + RefCounted searcher = core.getSearcher(); + Similarity sim = searcher.get().getSimilarityProvider().get("text_sim2"); + assertEquals(MockConfigurableSimilarity.class, sim.getClass()); + assertEquals("is there an echo?", ((MockConfigurableSimilarity)sim).getPassthrough()); + searcher.decref(); + } + + /** test a field where no similarity is specified */ + public void testDefaults() throws Exception { + SolrCore core = h.getCore(); + RefCounted searcher = core.getSearcher(); + Similarity sim = searcher.get().getSimilarityProvider().get("sim3text"); + assertEquals(MockConfigurableSimilarity.class, sim.getClass()); + assertEquals("I am your default sim", ((MockConfigurableSimilarity)sim).getPassthrough()); + searcher.decref(); + } + + /** ... and for a dynamic field */ + public void testDefaultsDynamic() throws Exception { + SolrCore core = h.getCore(); + RefCounted searcher = core.getSearcher(); + Similarity sim = searcher.get().getSimilarityProvider().get("text_sim3"); + assertEquals(MockConfigurableSimilarity.class, sim.getClass()); + assertEquals("I am your default sim", ((MockConfigurableSimilarity)sim).getPassthrough()); + searcher.decref(); + } + + /** test a field that does not exist */ + public void testNonexistent() throws Exception { + SolrCore core = h.getCore(); + RefCounted searcher = core.getSearcher(); + Similarity sim = searcher.get().getSimilarityProvider().get("sdfdsfdsfdswr5fsdfdsfdsfs"); + assertEquals(MockConfigurableSimilarity.class, sim.getClass()); + assertEquals("I am your default sim", ((MockConfigurableSimilarity)sim).getPassthrough()); + searcher.decref(); + } +} From 56c2994f660576397548e6981a1ade3e8e8be4ef Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Thu, 31 Mar 2011 20:16:10 +0000 Subject: [PATCH 049/200] Added a special case for looking up KeywordAnalyzer, which alone among analyzers occupies package o.a.l.analysis.core. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087431 13f79535-47bb-0310-9956-ffa450edef68 --- .../benchmark/byTask/tasks/NewAnalyzerTask.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewAnalyzerTask.java b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewAnalyzerTask.java index 6d301142229..7084125fa98 100644 --- a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewAnalyzerTask.java +++ b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewAnalyzerTask.java @@ -54,17 +54,16 @@ public class NewAnalyzerTask extends PerfTask { public int doLogic() throws IOException { String className = null; try { - if (current >= analyzerClassNames.size()) - { + if (current >= analyzerClassNames.size()) { current = 0; } className = analyzerClassNames.get(current++); - if (className == null || className.equals("")) - { - className = "org.apache.lucene.analysis.standard.StandardAnalyzer"; - } - if (className.indexOf(".") == -1 || className.startsWith("standard."))//there is no package name, assume o.a.l.analysis - { + if (className == null || className.equals("")) { + className = "org.apache.lucene.analysis.standard.StandardAnalyzer"; + } else if (className.equals("KeywordAnalyzer")) { + className = "org.apache.lucene.analysis.core.KeywordAnalyzer"; + } else if (className.indexOf(".") == -1 || className.startsWith("standard.")) { + //there is no package name, assume o.a.l.analysis className = "org.apache.lucene.analysis." + className; } getRunData().setAnalyzer(createAnalyzer(className)); From 3bbfa450e4e87f1adc316e6cc1207a6172fc3ea0 Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Thu, 31 Mar 2011 21:03:18 +0000 Subject: [PATCH 050/200] Updated to the new method for obtaining a top-level deleted docs bitset. Also checking the bitset for null, when there are no deleted docs. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087435 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/benchmark/byTask/tasks/ReadTask.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java index 414cf23c4c6..8c30924d2ce 100644 --- a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java +++ b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java @@ -30,6 +30,7 @@ import org.apache.lucene.benchmark.byTask.feeds.QueryMaker; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.search.Collector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.MultiTermQuery; @@ -95,9 +96,9 @@ public abstract class ReadTask extends PerfTask { // optionally warm and add num docs traversed to count if (withWarm()) { Document doc = null; - Bits delDocs = reader.getDeletedDocs(); + Bits delDocs = MultiFields.getDeletedDocs(reader); for (int m = 0; m < reader.maxDoc(); m++) { - if (!delDocs.get(m)) { + if (null == delDocs || ! delDocs.get(m)) { doc = reader.document(m); res += (doc == null ? 0 : 1); } From 4997068dc5ea86938bad1ec6220978ab7eda6532 Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Thu, 31 Mar 2011 21:31:08 +0000 Subject: [PATCH 051/200] site: sync news w/ announcement better, and add links git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1087446 13f79535-47bb-0310-9956-ffa450edef68 --- solr/site/features.html | 22 ++-- solr/site/features.pdf | Bin 20288 -> 14676 bytes solr/site/index.html | 96 ++++++++++-------- solr/site/index.pdf | Bin 118517 -> 79798 bytes solr/site/issue_tracking.html | 4 +- solr/site/issue_tracking.pdf | Bin 6466 -> 2304 bytes solr/site/linkmap.html | 4 +- solr/site/linkmap.pdf | Bin 9826 -> 2442 bytes solr/site/mailing_lists.html | 10 +- solr/site/mailing_lists.pdf | Bin 13709 -> 9000 bytes solr/site/tutorial.html | 86 ++++++++-------- solr/site/tutorial.pdf | Bin 52104 -> 47122 bytes solr/site/version_control.html | 12 +-- solr/site/version_control.pdf | Bin 13646 -> 8430 bytes .../src/documentation/content/xdocs/index.xml | 37 ++++--- 15 files changed, 144 insertions(+), 127 deletions(-) diff --git a/solr/site/features.html b/solr/site/features.html index cdd83f7685f..6874449de23 100755 --- a/solr/site/features.html +++ b/solr/site/features.html @@ -3,7 +3,7 @@ - + Introduction to The Solr Enterprise Search Server @@ -199,7 +199,7 @@ document.write("Last Published: " + document.lastModified); |alternative credits +-->

-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
BENju<+X^!+pvIEgRTUYbNT2N(x!pLZq@$d$4w-kv-v6*~lqT^gE%Rk+m{x1-hpx?r;Z3ed^}Q z;#WtQf~2qM)C^y&g>=eZRQ(aJebvs2?mK&&(4^)A1-x>4ZU zYiiSk(5BdFTPV_TE7~qr3v0*Kdh=TA0;yV1(bI61!(uQVY0~XOadRd^S+~kyu2PL{ z2lly1O5Enro||$cohAQn8_)3Du0^s!DDpXK3%Sah^Efu+Rn{`;(d~q5CC@TksT(cj z#)$$J6sCA()dgpxbFdWL4=X9B(_5iRt4X-;>w(8#BgJ-GZU1O+v}RnKn3T)&29$wG zOTKV!Q|)ina<7vse32N21c>5WAjz=@hY|t+SRZ+#ND;;ZeZ zy|k6{7Z6H`45B^oMn8I}wzP;qso`-dhHq_6dq0#6x_H5TT!gp7t}Rt_$zVTR1c+e- zT$@T$saE5`jWV9J#p(6Z&TdMqIL>GMiqGdfxUlJ_<*IwstfP6#_=kJgO%5m5X~pBI z&Bf)k=E|1EQzzjMkNUT4xEg20WhFV{L~&+;inuV_yJM z3nN}j_MG(!7bqL$TT8`0?S_hL1tmV3T)puHZz0hrw*jYa+p%`uY*KrMXGlSk3iW|I z6iwSld}2hot2dfi^%*a{3-*%%MPdetK|!3TV0;>=bIQ|b);#0=t{sZ4ch1hf>$lv5 zWiMC?*yv0GShVMGqt2P7vzwir?m!pac^!d#bT^gRPL%Sl6UrwTp7IolRb_kpXmF1$a8|=YX zClyzMIQysKe4R{r?Br^^Z(70O2CvQ%UVby#Byyf?w!uxi-oOsZ!dT{uNVA<3BPX(L ziKR(#((Od(q*dI|OVU<49kGIh4(aOp8l>_f#Yo=GaEkag5pnI$tSXjUqR}SZO3AV8 zv)q({a5!?>m#muw%7vGF2PP|OpcMN}uJ0`OmEf(jnJ1fV`4kLA{Fc&nTygSgB)!bp z2Mk(~n+<2R7>GN{))-pw`MsRWwt(D^Em168IA)Qcx)G10RMG{(KjL$VjFsO7M08xL zctKEmv@>`)WblFNV|JX_&y)Dc)~Bqtm^GXcyXty2-+{+0iO!f{W40J^#Yo7H5xeFOavS~VQA!3QyGQDEgTB(*G z#L=TjO9XP&((W`DN*_75b0HqCl=xdRRLqPc@J0Ff5&<3=&MYl7hA5$x_ zVD%iY5XrRchX)QW>~#WY7{E!L!ZUF6=DT z{m=!9wpKx)?m=;oJjc3;RyDGT)u#L5wB~c$%NbF2P@I*minX#-^VE43trQ4lU!N%H zqcU+^gX#vG)h2@1-#aZ9lN+}I3s@*qlfceboIgUNKVR9B4G1?INtXhSCk zlV&1(RuYu+kWMfiqCJaBJ@B4xbQTp!MpCRKMz~$@Os`_FOz{wCwI`Pv4Ciq>^1Ky? z(rL{+SE-+d)^N?gmplGd=*C6@p4BM0AP#|xH#-`PYH2i`z0xd^_Ldj>Mk|20XTI^3 z#j6?YP{}ozkvp2`kE6%dP{k_2yqF~tR~6jd`vOMTi{WV(x#87XIGXWYWl2OXqf@ZR zrxRy;pR!EK*AUEj!9K?#MJ1C}qwO}zU0hBtC3e~Dmd>$Z8(JAf-4QTer*apy+f;0* zH)(V48fWm)7l&8BEi&+nD=^sZ3YGGuKqR_@CV!&4YoU)MBQE!Pxw|C_rht=LHZH>F)b*YC1iXz<{?)mLN^lUz|9zsr( zXVc4{vL5Q2g*WRVYgh>1tcQAowwoLO06U0o4a88xa0vCd@7IRu5QCkxW-Y`}!+3~6 zo*Cm>+mLAOhG`N*jZU!F8sqnR*cf=0y|XqN9e!ggi7`g39Sz>C5GV*al5j177z7Uf zyKaW3<9`G-a1oDSH~B&!q(Q)M;H9~jFt95&@FKvS5QP}se;+R%3^x21QTncb(0!;V zC|eHh$TS#YVAo7R>%Z$M`RWuBR-ptKuWrFW^NQft97ZtonlcO&fRG-!gEKs`Z^6!{%KDI?c(J+OFfa6~b$2Sd7T4Fv*V=Yv3 zH>SW_T)QG!&ENlbVE@S8{)KW77{q2j&sLwi3PLtMJ=$JNR0#$%@P-8KMgr}BPiucj z^W=e-|NbBFpz0j=!a@4^D<{dn^s6!^u5Sp2o=-sYSk>~}X`Y?HRA3N<(+?;+_Xk}& zjnUvW)GpGm6RsH(?>F$!kMCZ68!-(DEt$X{_H8sH!D*=k^n3!&zj!&8<`0m1s-e%Z zZJ1+y?F@yYpUaNEwD<<5805jzlBMEWN=R+QH1jPam&D=x?oT>$evYi5E}?=TlqLCn zyuW!!_Y-f3G%;M{K^9>m*2>=hNf-G~@fKfQ9Y4q1dDgf==L2o2^>SH5%Ku;S!8eyB zI>+Nsj7A%D4334MP7Q={{8`8IeuRJLqWY8AMZN||M49lqtEf+z|-2D zl_32ktbNkKCre8~z5eafI#~;A3C_jg#`Opv5qu~0csgDr?M1le;Zw5gQveWN6?`id zJZ<4w$hpbOL7y>B#^)pfpo9MfMf0NW`uM=G4G+^afWM!c=%AX zwji&$vx1x*Re^AaQq~Nk;)H)D#|Wjq$@6&KRiMvT3@QkZ@>^&A;22v&tmmk#;zO|8 zjB|7|Pj>y4c5QU9itV|5-7c31by&V$*QY!eE{03&es_#-t$5yfmRsI-9cn*^4z^DA zm{~K^HPp(?20h6w>AJ~>_Svs zEiGgLSyp#^c2CQxTK>kyWp5c9MiVE8cVf$O1lRX&Tg$!eh)B+;qbp~6N*BC*LRTz1 z*f3FuNw$UgDZNatsk$Frb3)9iE7|4)4wB0Z=C#1QScNkh1Q-kJX?+T zd1ezCW2n1*X2m78qUz~7!e&Rg24+p;(rW~#*k5KYBE_1duUQF5>Bxw=UTd|NTz(E!(7JZoOv3(k$TM|WG2m= z9ZT%QmX%&+i1JOZB{v(D8^c8s5_B!eq$LR6ylzAal&^ufu9mtjd54dRi5ldMdJlCL?=_+v9WBDBW@8yH^3Hg3`?+FM8< z=Hw#Va$r4(x!pygo8v2!T42>4>}|POX;B%UgYphj!gwIfAQQ)BkoK|td>ZnT_{e41 zs~}vn>hJ6%j^x}*Hqk(%v6G9=21cQNZOt|Op-Y4dF5~L4ZXxm5O$v^7F0hBlnCSOG zX%8Wt8c3AW}AZP2LsJI9FS^$bECHWGfyWAkN2}gaml7uyj$r)=OIH2yO!`Y@&P->vM%PfyF6q+0$S9c?G3ryO~ zA}&ORn?96yyaZUkpDRjSlUA0QYn(YLe%k;vn-1u;4 zow2QV3^g|cL~ZX#lJojT!P>J}GG;Gtz4dX4^~8hQ{WjBS2g7@1LVJr%D%EtwG8UDa zVP+ee%Hk&z9<%sv=@%E|^S3Ei~y8bOX>hKcOW&Stb|D^@VH!J0!im|=Ir8AOE{ z*LTAiemBJ7cS9VMi`MVsMlV0z^^6aFV;QV(RABi>Yssx)@eAfO*#6R5ZtL%DAr)Sd z!L%}$r3++&{$&|;Ovm&afRg0hi~`{|+HM{4w88$0g5byB^DX;u&zJ$=RcU#HwM`q= zTPyJws~5AC_S1bY6oS%yT_}ii1dkWkf#Be8Nr4BM?e>gZy=yqth9*IzO|Lz17N@k;DF?Txu#N5h!jAJKT5ay~{w>-5eF zIxKf^A7Ej0M)?P1P~$-Zjr;G`&hi9avuvrDxSvTzBGOui?>#v2muAXPqYrZYeiR->;wNh) z;jB;BqK`1VO)nqA@J2K2K`wRmf>B0`&JjlckL@3E$0J-nRk}gMpJ3jJ_^R*$YLloE zZyTczq1IK1!5*kF)#z+M>a=@QoUhvNjhh}Gc~q9hs}vY_KbVv52)qva9f4Qn4+xmW zP`mjFf9uS9=kH%Y^ai>vpBVNTh_6JwgZQf00YtM1YBxUtQP&T&#sPOY9~6V3!gU6? zkHVuE8o%|&4I!lsp4Qvxk0;buIegXUcb)ipc^`pSlHbvLRgv#>;+@YPWkCDNmoR-6 zM)VtK{Up?*HYE)S)<@2eox21*e)!!&2+q5Vg-{YUX!_4bOnx={YM*^n>;~a=Wpba0 zjTU`yjJ43?;23I$qCaD?2l&i#t8tk@X;>%zeG(u709L_JPwlq6>X%w0_5-QGc^lj# zejq9SB52RNAO%7G3*uK+d?0m{*FTW#VRc-?F6IU4ONZp)BK>T5Fk+C}B>IgWsmC5W zk|JCJts?{(eYHFDR_Rj&0o4x>{Ay=!^T(H{iM{I$ghG|TN$5Moe48x3)FJzb-MzoB z;`XNDztk1}*wq`S)R!_;+!4Oo>6=#mQm4#grv|fpgPJDB)HZ{cANLg;(5@`4FE3+V zo*-;flXdVxF#gfCHRRvZuk<{VEeP4uuXNYE7VI#LN2f1u(qQJ7G`LgA$L$Z;XF=aD zuZzg~4LlYPTEp8U4G&b0YmmzBMR>jcK82_Fh{`3)qobsQQ*I#FKVG3 zgWr00@SBC-x-Qe&wHpRrNDd4LcZ`u7oPLtJLV-SC0ATgFY75twG0exA+zU8S0)S~4 zHf?{XA^q~>z9D?mIeOj3{IPTVfn;9BcakE5;phd5&!Lcdl7+tyH(8!OL@VoCwfpRV zv|o^aexG6qJy_E8g*Vdt0s`<>(hs{$UwC88_g{FW&KnNkVn~~Dc$ned;h@*qZN0qF zfM9?Ir1=B?M8rD_zI}iOlS#<*zP|YVh=`{5aa^kv%b36LPei`YP@t9PWbm2EoUSq%X0)^ga&BthAWqX9ym5MelGhL7=6>(t`d> zU-%~~-nGUX4kVbsK6wA17}4}T76Rv-52$Eplsg$f-i^KPPfN*tU1zM^-c~mMRfWwD z&Hau3f&Tmh0!tCp%jU&sgSseMb^z_q%i(7mtI6;CHW*;gjqlpvLOTF?Zqw4}KDDu0 z>cIDH;5KY-!?h!mch5m>QLRk$yEYUfw5rhW+DHP|?8fifP>j^Z_+1-3r)64r*9MXO zn#O+D1~vG!vd!<>P{@5}?w2&di~=K>`z5usg73cvPIBgcDNSKNJqOOGPi??yrZGan zEA<&Scuq@O{PaC2;QG`?LW%=3e3WV2NXQju{vM;*Oy2bY&zboE%9Wb&Wwj~g-Mc8x zYDq2LwZTb|=CA&)4Fz{Y;Q9A$6#YRPW8xQ-Vl0PrdU{G}L89p5HDn5@n{(+7F4QA4(IQ)600YpTY zOlR==H1RkWP4Yo}ptFe&WSaOurU{Vg zXM71J34Z*Rq={edo=Yg8k%5%N{9t$&wsST%|`vsAKYH!c)lG+~i>3a~m zVQPaElBPEETnn1cy!Jr}6K_Cabkq6@{NlB(&^uUwT}*Qd2_a<9_>yqW_Y5;&X5v#= zU72CQKro;C{Xln;kj29UA1F@~ouSmB`8m?0H%I{4{M0umx6v5M}HknWBJEV@}pgLEUI7=)A{NJ}F~Nv9x)NSA;# zNQwS)f$r^l-hH;ZzxSN$T>qu6IoFtTjy0Z<_kBOlm|jzslwt>Sz;LhSJ@M2Dk?$B$`%RyFsGC)kgp`d z)WHma%EJW-{G1)GdvZ#mo4VR4S!iD!Pn~i+VZ_~j8%tFT8N51~I1W}Y4&lv)STuTW zNp(j_mALG45ay}dnG$g&^bx`6DkUqJ6p}+uj--@{vStMolP^5hdy*z;_Izy~*Y%BC ztosbtpv8U;JliC%jdS}#x;E+b;;XReNDE_Z^a~qkXKc}?9G^cIC)=08c!=TsHp=!P z$v$~q4L!ad~DaaSk?N>wU8BGI`vLT|a z4s#_G-iq94%<=Cz7F^WKY}rlYnIQq}vmR(uG95-6uQ|25`$)P7UiEhCau74*gi$c+ ztA#`-kVH=R^(ibyzUl_y^F&{r0l$4np+hz8jAq{VS%=iaj-r%a;4Lp7FUXGFuAX_w z&){=X{NAlZ!yvyk3Q3y%=1*F_5Ao27vk~Q^hoN>>7tst>(3NYCLwk%l(IQIgUGaUK zuGDxNs&MsU;qKcByzxk|gdvZ;xo5Uy zOMCB(F93H=w|n=_!_L8WAjz+wl_8-*gJbi_PKRWsLbAT@{3%dYfefw_B7jD7c;Q zqG~9TR-&D%G*~;9a_-Do2_j{xyXUS+h|6GH6kSPEeM*%VvYc-ILGJ@I@^TFtoIj**oOV4@tQ$AL%JO&5chX!8 zp284TOm`i^0RE!s&V?3vLPB|FE7fjI_1rH3$DQRBJ?8m zIx7(46L@t7=~%&Qq%Wu#cD+SoO4pIeQOl8}isA+F1mSuV!K>|!u1&+HNVDch^C z8Hs6?2<9VZ3uaUGpgbjnTJ@0YkkqoyGB2J4H2ZA7s@0P+2g|@RmWvS)DL2b*rrwP1 zbh^kwolZG$=EKqspiJ$AiT zvs)8b1NCh6RPhYV)HI*V0+Rf)G5!q!aMY!GrBQWPB( zZ4$+O$(2F~q6x(Z_43$r)!T@3JMrk4zczO6AvNuUC_u;UJV!5ZKj)DHKLb-jLVKvY z&o0#VgtGUODUyjtVntHQ6AvalY_>(j=)PtTWIxYd%l24@Li?4T-Sg$jLrXh{6#H>Y zpSBPa5=%A9LDTJL1rOOrDq26i)8EpctbVq5tIx;Y!oIC?%+zf#)tbiSWn=xwm%zZx z!1nYW`0IfN@IcA%CRSuCuBn}QikrBA2VcQ0{)L_4(MFw*-fs`c$+P7i$whW$dbDou zZ(m-Fu3oNo)?YDrVt=7hva-%z)y=>y#w+mX41o_ps&qnIP)@Ez-lABah2gLveGz`q zh^@t^o1a)5TE_ZnTfF*1XGrGNR$V5w`glew-?(-5O*jNkviOzx#eU`fN{OzE{{HsD z?Of5ehI{@#Vp(Zt#omY!1oQ;lXcVGzlre7q6kEHEw|IWhEjXb6cK>SnUQ<+aVz3WQ z3=YBhA{@UgQ8)4s(W|)GMA>7<>GQMToIZ=*<1N2TtlTMAoyt;=%wiuR~RFoEOs7h^yfW!=Bc$(#Gxxx-|EogUmjF$W#h+O#=*wvUCm)`pcJP$ z@F=5X*LOuP@?yk|S3KNRwok{x-Z8)Bt+HCn&QqX~bH7F|S0H!&=A^@def|Eu9<6&u zKJVho5XIf;wtbr2n$yE8WA7`enYB3jIBCYNl-9Xmn@V@Y+!wh2@r5#6OQkZhSguq~ zmqLHk0HIu7zUOTnwad^P*sNUcY0(Z|iVgPJ)STDp7oMaJtro-Yr3u`)v< zd>;7l9bMs;;`!v)b68U?o0(g-)a*O$yW}5s#-}Hyr-%I!JEFkxS(fXTaZ_X0q`TkX zc&qI*hn&j$S1>{uLa+EG?XT`Oi-mgKDUK=gvDi1T!!FOvB3ag-;CplL^|jTQl`F!R z-ODQ5p5)AWb*=Bw3<_zw=+C>4;xvV4kUUmgc$_SJ&vkieX>QQ8PR?^ZIb6u4fb)gJ zhHd}o_M&%R%F^CP5s!)8&n4%3$zu%73_jM(Ip(@F_@>QbAJOn*QDX^&QiaxV3K*%24)T%2=ttE3@eck>!ZhA#!#9>0FimtAs_TYo&VZmahi+$gF9T#$+%DL3n zbrDC8_9%T}$0Y{Kv_itmLi`)Xje!ED?@O3O5MOtEUwoiDfWHdrmE68MCf&y9!^A0K zxbOa1b~xc-&MT4YU-gf==SJOLx_^d%E(&mc(ck$zSd?V+^yS;Huh$Uk$hSMkS`B!H?>E`Cf;RfY!aJJwC^YQU~GyNicC67J4!?{f8f4w48LQ)eqj z6mI{PlbPus89BN-+n!`&X3B}MMc5(ikuIEI4lpM@E$Y|dCJwGhA$n9+W~Tf|6qEQ? zE-_o=w=-KSRJQ!KaC-|OdUtj+ggM;R7D+GsFN8yo6P2B<74Xc-+3Dk)KV%A=n*CYQ z-};sRKJ$Og5%m)Q!}ylB8p7!h3xrWjh8+ZDhl16?Abtph|Afx)gFyc^@6)GH1^<@S zi8(*i@wYsGNcmRljz6@Yy$i=lb90zF*l_|EfGUUndzAwh_yJz`jf4pU^iB|U^N(q` zf+q>zxSQ~QL*2eT_WiEYhfc_xgYzHHI5;~aT#&TvwBP!e`Dp?N{DcmkbduA2zCZY_ zDSxo^zqK9c?0{3{Rt%e{YT z)JYoyBN4EV^N*1${O8CA<{Tk2Oor&mWzvz14au2@p15hctH?aE*=mEgw_Bc;lNFxlRv;0aR+BJfI9z> zB1S7HNXx0ADoe}!!(`#$3^%n!oOCxN!qn0pfpl|lw&Apf--4ffxF9&q9Ng?}9pGjz zoL2T`2zL%kq}_>5aGDt~;c?2)a?0ZV!0y4&-!lyc=0X|s-861)4qiU){|Cby91-@` zW;PrsD>wj?I8n03-xT*xRpggi0>%S5E#PTO@Nsc)al`%}ii?`GtxW7r8o>;43tk_BKe9|t$8%wTSy?|^xL>IZ5c!ovaj#~ta1*}%-hp3~IU z!3E*WVQOK;X$z=F_6SZFKxg`y8Q&LYR8Re`2Jr!XN%b$aAlg3{TR;oa)^Qo*CiLAa znnLmqx-*905Ms)Ta5=?0)^NQR$sn&(z&X6NJU72J*Bo27;oiSNJ>zTqV>ohvdkoDqOFGQs-#) z{WbjLPj!t>!z)8Jn7HN3X^lg#xHliR8CctSX8TJE-Wjbh#hG!LD1C9a_K1Ge_F1pz zWkiMjC`wGsp-hpc^UfPu$hrb9$6vD zUbHTHmtkTrgE@1QjHE%O#9Vk2W7l-UZE=4r@oE?e^7H*M%r^!dPkU#m6sTmYh+jS{ z13NTB!X5Pr2h?3tnGU2V^+P=jK8?XA*k{(^6>@Q=IH{dYIfwnYdx-2i$i>iU3%x?T zo9D-}3m{W(`W=VQa1g~l=ro=nsA|m#I-eduaFCqwq7*CQuLa+;rYQwRvt%+TzjkXYYN{=+jU@K>58NzanJJDq-`#R!ecRx&loKD3 z&})^laIiXw{*G)thncykW>P^g+#z6HniyuoO;$fBM-eM5~BP=kqoVFul*$ z$m@c675{Op?`p!Ld?T2&;A+QWz?tfTAgXPv_%qM5u(xH*bqK1g;%yqnJNaK7GZZ2q5Au;Li8pWPf?{45>h5(6Dn6J>yUn^Y-*ZLUNWhuHpg0TNRl=&BcPHK!>OXRfWD+%kJXv8e>| zPGxm8`(+C@UmfY;6@Byme(nRlHU9_AShgY)|sFDSMo1s|${{7>i{ho~Ac0!Hv;vZdb(PGNO#cZ0l2+=*T1Y!Wwb# zQj5OW*Av^+Q>*MJVtH+Ij&Rv^Wb)9$1j=bSruA~Ab6!{pQ57J(y}iWJYy2U*S5zry z#59GavLr^6TF|e~Yn$h_{)7W9MNj;Dwuz8|-I*p#wnm{bjob4oZKmSvf^Rd~O~5Z- z5x^-oh>(GK;a4zfxE>VySkm!py@|yo|9a0j#t1Jz6ep=-PAK;ZRx8`<*XIg|R0gRJ z>+j&Cl37$=Rv+}jcnJN3Mit86;3Mj|zW9prd3#dQJH9&bH~(xiC17#gx2=;m!?FY=Z};iI*gLFqA=WGTWbYKO=P14dh>l@RuLZ zWQ>d1AM?bhL}0Ko*hFiDO4sQ0tBzOd5QWTgBV9J#>SA7dJ~?7Yani9gqk`F5y?en) zv4T8{I7sb2N9ZW(L!N{s^fDWsBCVIOf9b;|!-M^g=7+v%B7Ewg_yhjYyD}T)90Ltd z*JHv*0(tkzQa#JLUS}^Eb&M4AQo)naOV1f;)De}`)IP4DLBIJv1b()LxhmZ%6HT24 zg9+c|J?yAYBhmPlV64kf$YziIpOrWB6Gb!Qorv?j z9QvXA#Wn8RYj1XM>_s0Qso6*I7pWmiyf?QNu)5R(N;^y>AP>&lH+#i*;_0YAyMF^+ zFx$9;j|t=UrRm*beGrw>{9#_xzC7X02M^%*8&CXe-ClS!xnY@C$AEMn2GZqAE+>8t z7(7pIl=;HX<^v`7x?uPg)W48}k=38$iSIR4(RB??r;@N{417gao_6A5e-?QI5uYV8 z9%@P<=Xr2_JS*2CKE+tuWL^8F&mL)Q%v}2g8$AviY5d1Z6UvwEY)G8OYg(Izt_I+6 zOyf7eAn3OBtC}uml|db?4>1%Al334lHxlc_uW;yLFR-$>N$4k+ja98M@W?jIZ+jKZ;h zl5kaoi-W7PDZ+&o1uRcoL#Mo48E%0I9X$&;pr$OOSA|N22&JEf|GYR8he9yLfQoyFwKl z!o|n^Me?zXt{YI z|Bw;bI+((3Egf8t{D3D2dJ5U#W_DKgKLKu#-vs=8AOLAX!BF561_eHOX`v7(2k$=u zpMO~>v6mqctdtLzfkr#Y}qmYxH z`g1tiqcdjzo}18TPwSji+*`5kYp(avGi3((We67+@Ltp7)Z<&t>1j5UH|UDu6o;-ZDMzI zO$MAmS#eeP`5MVX{0nbr!NRMg#cQv67db(cZre5`Bt(~XYsZ|-lbd7^hH$k>!7?8*r8z%VNg}sTw=gzM)rYiQ$po;#dI_NwCRat9 zMhl|-inB#{aU*@z)pEI%#0x{-L4tGCt;R`j=<4q%Esh-1n!@w5^<4PR=lSz9GOxW$ ztAP$(pMD{z&juk2)@ic#{eWFx{&f1ojZ2GGCH-qllD?Ow`XSW*E&gWWu@w8aWxL#n zf+(i0E@E<53fL6;rWx4OB1gDvvP6b=yh#cPFH!0^&w9KXCNv87M`Q>-#djhj>pP;)RMYW0CgMQVE80i>!39C+}`ca_IbY`WMc#;_g%F-ITdHm_0DYE9+C&i6t2 z3wYld>F;4Gj2y0QY3B|MA(v|--t3A!45rGG9jSj!6j4g*ykYCd!6K;QCaU&K30H@# z3L~dqZn=jR!4s?8U#>56{ey(Ajf`YPqJS8>t^$na+8{4iHVc79EzPREbRgb*7%!oe zn~7fNXgKy~r(^#8>s0iY%I}PAkZ8+D>cmFifoorz<|RDW*A+?_sCiJNnsS&oe_{V^ zi`dix47V+J_>QozyO7shP_AfoX;4l!PiSGz75s;o%GW=hW1J@Hf*OD~jAM}MO)bJ( z>09Ve$i{`xIOtMCgoHmF>=(bP4Y48$@=@v!(8Lk-blq2yKK4vz!*V~cQol&+f9!Pj z){W6%8ZUZxAYyXwBr*@MiiMV z&v&-rPS@-iBMo#k6)t^(w@EuBaLL==lOyx$k5xf83LfQQ+$gnq;-OO-aUPFx>pj|N zQ|*L;B1J+LoSUW7rI|YOs<5Rh%QTV`jr8Dw_MU)GKu)+Zjs0~(o26VY+*EoOX3E#= zx2S~tyqve3JKFAJ8yJuhMaLIcrIl~>%8RvxoGl9%rDOQCa080N>H2v@;sDpdo87c5 zTsc@WeXNWJ`6Av%3`ZkcHE&#`muOe>aZjG5j$&KKQNs`{KSNSVba&N&NLObq$+HlD z9bZ)~F?A2GpwR7}Yd^IXJd0lv95-ON#qb~_;-YCx?0uO}h^Eg@hnm~=RMouJ{; z2u*nbfp}ob?M&Gj+{MKsRMo{7vKTK&QbT(*-Lca2wFc9xowEnu-1S5>vd1K*O^kJ z5^ikE_W0zdmv=xNX!qE@JdbW8+iVQHq;mZWB$_3Oe0?f(lm3xIoE#?8R>{(5_xkw< z6K>KaIooI#XZ3{s(<2B$p1lndQ$2W!1@P1JpnNPf=^H2 z%PH#QhW&v$|8#l%l6nXOK{+720M3L$IJjwHe849Fj=5n_0E7M`9oovq>PK(FiBgF2 z5~;W%oITV|99KU1288T3Omt4xxpa7ntxLMzek|A ztjrJ&e_wp)pWW$qiqFdhR5+L$%E8A6xB>W3Qtp4g`1Wu+1plwt@SD`-TN41p1w;!0 zIuaMqg}8XZ|05*e8btiWg6qJEqqCJM;u=&;kM+8`(sdXJWbl)=`92x^V%LI#csRHL zQy{!RbHJb{UK(01;P?-v^}iY$fb$A2Yv<_TjFf@fo7p0qe`3OKva@eZ1JE;IbntR> zK!JIL3j&Oif7HF){>lOkN0gUTLQV50CI69u{qnem0Rsc*R6vPQY#0XkXZ{(p{F{>B zR8vy?Nzs4PB?IM#aqw~T0wf+a4Z(n3#K*%0cx7O~nD`G)t2f+{fX5GMWpAO1aB>Cw zy1&+gess+IN;^PM{v}!%3>eQK5a6T)bHO;EP+&lCp_Yt)GE4nsW1toZ>z}m4Z!T^p z(gcJXSQ$!_)GWE?&Re`51$aFi zQ8CUHxVuLE7+*sBGJf+Ld&2D*2d~7Z1a2mSf~7Zoy_w!Y5DJVp>9rOQK+9`S=dlg9 zm;G~Eb_VX}K4z}EL@Gh;h|j*dPvm5}KdW=GQn{z$yqyRZdC(AE3K}+Dx5~5Ewfc90 zs#w-|TyI5S3p~WaZ2*ap;)Qia`1FcI*685G84<+$)z|4E=5OtnbvWmGjoD_5d=8&x z+)|%YyAv>ZX6EzbP!Th$raa_zz>>zx6qGBZTy;}EA()A_^^$(s4) z6bBWHAXAVfX6pspq)$A=;@S`KvG2|}T@s4aywceTS9j68CXkyFOAHE*(^bc%H@VvE zf$iKiS}}YRh8w~%SJ+Oi%C9LOrppIirRcK|K@qwfXiqx6JGz9_985j zs+c=+&T1N#S})*qCaR`~)_^abOdMYf*jcI6GKm9zYKHM!Q+wRTaX zC#iZm%SX5ZhjTtu5ib{iw{JuV)TN;C-F@AP>Bz#wuIPF82-ZEcHPFkEz@5`sE(#5nH zlac2}#pf|osBtxKzQ<|Ej;e-@*&2JWY#z0zBFcDF6^yykuoT#>Ic*-yfXgqRsgXeX z-52+5lT^E_=0~LvA|Pe>fX&O{ruwF&dhT6MwQzA(hxTgo^3IELY`4(R@$vkrrVRZU zn<)#9Ws;3<$GRXUk0N>2{izCKRh*L$4=*jF z`Py1I5m@J|N-@%8BdOLey7YsrKcQp;FI}FIwiDXRCEoD|!iCSP5p2fmA(EuQcZ(Lh zG)qF{s%f+*whq5+-4l0-ugnc6PVaAi+&^NyqKoxrEGMW-KBB`l2A*b_a3Cx+>2pvu zM=*r>ia>Fv%}gi}97sj8X8)DNRFuIE9d7ve1D>(Ef}AKQ7H1<6{FRuUuOXR{o%-DB zQt6)5jk4aG*`-g?#nxLiHK6jnK4<7-UGmrtKQE*(ROXLT6YTY0bs{vG7P;5;RL$N! z^)Ye(0vunHUU|S&R3fbnZmc$vEMl(+BUOKNH4n3$>!Mt7@#oI;!lz08o@p5mnsH5L zl^@)(f5>)JF02?^EO^emwqKh@k!9j=JhebVWFu^azul>I$7%MG9;0)V+&IBUe+nZ0 zuHcZ!1VLz8w)0p$@mR#gezR-JJaT4?7qKCHXlmt^3%GI7+NCenWA@X#GDGlcJ-c5S zUXKvMr8J&0^%yJH?T_yozTm|XL0xQL`Ti9foR4VO%a;o8(jNAK@3<*V?uVj2q-tzE z9rCto@w33;oRtgd&R~IN)0U}sRd4R|eo@$EJ)1x_UV?4ar+T~O;yAYe<2IY$)RRXS zS8wGAw-?j|Qy%8JzfwS2Ymc-yYMK8$Y<%MVc1#u}u64DZb{?3s?PX7D>uWmH^_Xs+9V)O^o4YFU%H~atf#BMH zl@6>1ei5{2`OrmYqTtrUAvL^VL3lyu{gf%_s{Ts{t&moa5AWyaJ7UQ5D}6;u7G*0s zs$PgmZF=aLi+X>Z>Rarb7rbx&ico`FSOfzj3In@A-ohENMD(={T$0kB{lzv3(#KQG zJTmbbe>q{Nn@zitf;9G?t6DjZ@KX1O7H{VS7q%2lllW+JoPJEwGUNMFbTF4{!@Q5| zMJiX~8m5(W#?{*7`Le7s72`5;GvshP-bu-`RLmSDU&BSSE+8Yeyy6ses>dX-u73xJoX;9k%+R;gt^G^Y(+D)sf0YEuo&L`%gVWU1jyR zbMDrl>!a1*C6ec9SqlU2*>>%Ab-r*eVQ3)(;d)3;-t?`R; zk@+v0R&2Js>O|7Z8fTAqNNs1*90O~ruTSihQsD0mgUP5ZCo_%4)Ucjg6qIHM7?`Cb;%AU9}9Qx z2jmxdVt$w-y(QU5p2(14tDQYtM|iK16gP7I{Jo%+8!b4s*j2e?e&{R5yTyb0G9uWw zZUw(^vf&-0oSj&c3tpPIFHSbOzy$u{-L7O>I8qqvb|J)8BXx<=5;3c9D{Y&)mApdc zsR>TXw;lG6Un+@sD8@u3{Zhq7;{_qXGWV>HCB}s5Qszw2Uc)&TamY#{;sS2US6B51 z33`d_FQPnDcHd(3Kon0nH zu8q~rKy1m&gqEXo(B$1wpQYNfaXIJf^4H&9&HtK@n?`N1jg6LS?$Wm;J|DKOj(Fq8 z1GR_`rV1{KE;5Ip-OjruWghyrB`~Dvj$l4}PR*CXq_-pI6DcMFs9Ck-_UOH3_cjTy zN}~HKo*U;9dP1}xbxc(`p`75dAx1coaceyYS>j&suv|Ioo`^E^MKuj(ehzkay=K$~PKO!@K zWyYz`5^z~snenSBL%6`OpD;|=Z<1peD(DXBSvrodnT=L9+}l<$AUkQ3=hQc{)C z+*Sy6{`VvY_M3zm0(iUmxKZR10%#P#Cm$fZ03s6V_#f0-PL&F6J6i>Wiwh8L;PMkI z{v7MSB;`T4K^#yJ3`LQFGr%0+49bN%1Ef8aSN|mCKS!96fM5i7Hv1j&5Bp69eJUS; z0r?13crYyl2vr6H@w{L_kb#_}{1bEj`yswnruH0&n3Il`%=l8cdxm_;bpV^#JWgH(z)iF?-y2HS7R&cTMH(>+m3teZgrXO zt+YR9OMQ=K7I{)v-W_bZt!X5&UYkTg!tgoLcb>1 zOSr=@Ik;QMqIoB!cp{M8q$DV_M7#G&@B@qW;Zbp)s+_Jh7Mr;?0;Jlaj#mHWjzI*A zX0r5kk=R6-Ui{TTX`@izrYRHUsHKKa51s^qoG#Be9=~qcRSRXuh$5*tH%?kVoZ}QF ziaqk`nA)CZvhe|Bds$RpK~!*D9ISzhMOJI*`T=+a!tThKNPd${UvN^w3D;ikLQHyK z9g+0NroYnVrs>obzYkKyI%ehC^HDEh_Py2H^WIEkl}2k8y7jip?*cS6G#4Q}PxYB; z?1x2leD~+q5*V`A%4|xR69sxgC(CCqsw=5Hkm@GXPqEXqXHco30{1MLsKwPBEYpK<++{Saa&WH#|o}un5^$kMtB9qC4j{Mjc<thd%`sU7TTE3$}5MsbR)7;W1+-G8OqmUl9f;*aP_+BnEAe zZ13!XBBELp&fb0Gaef2uX*`2?v`A^eoUca@WJh5*U$?wDi?fEOob+hxj{Id;g)#i4 zJ2_f)Hyw&1KCd3L-*((MSiYEFWEjq=7_56)`NGN!8mVavo>#T-Ad^^CY)7RU+Vczv z7uF*ed?7TRkm0wx%9ulR_94md@X=nbM%=J@kU~-gjmTaP&bQS?3Sv6qUBilC#?Qyo zl--lC@OzsWpr`&umE;8(o(9g{!5q(`Ws8^Z4PPdWf8Lv|?HhS)zqWP_z0~-u9aTSX z8M^-V`}kIQwP3<`flK{?;1>1nSx1?QJ>R3_HEG!|<@?cl*qDmqVJi0@TF9&;#S`9Q ze+;?m>o?ct)LzD5Lq@!lNF25&DTkr>NoZ{jeV*Qr-Q+0D7~FxWJl`lxcjsc1ca#Uy zlRYC`irE-;;y$7!Iucufx>Orv0M?@TDo1HF#;1(lj^~){Xx$enOuZGZW!eppZ98M6 zIT1_D`!{^@+~nqJr{h~DOh{8pVCs^0^A{DLMU&LmHo~Jj9k%I|8*dC9{rEWaK`DHmP!n^XZr?FVgfH(GYyO5|S9V zYyV|)YF&(yhJh+eYF{LH^@wQfJbf4LDK5m%YdYBkr$eM5W_+CmIT14nL(1e(;M6gH9 z+pI?OKWqOQw;(LKG;pvwi*`df{LadH%!h@l^$F*=#!{`K!@f2y*LoVgPwoz4g%M8< zo>DON-5$3~C&sZBP1v2D#4fpzAuiIekfovX%xOj`BeDE~m;l8a_^a{!piZArUifE1 zy(jj)%2t>-Q*PI|AGY_Lab>1#CLD0&_Da6lUrN0sF=C){@hhd!Q|UuV?hMLy8x7dm za6(e+LHAdciRPu4HgXFd@O@Q*NZ9qj2x~)mH$b(+@KLc@?;sI1$>ZE4Ilpk0j)e*{ z!o1)`N%uQN5&GPCc&TMcNDJZ1;d?39A_TeEG$QhD285_1<+e7_BK$wfJR8FLDx?-y zcjp#M%w31cZI*Dt^T-dEL#U2uy$`0%RKt|f6dZ`rr#$VW=+W>^w?Xfh&sR{9oMms| zrdpYZ6IauHW);nauR_EgQcT{-exI<)bRGY5u3jx`g2ff+S+iT9r)v&3qNnff7LV;s zV5|z(4v<}>z_i{obxzK6YMkP~Uuc;okd~*pGPQQZ$1zuexV}O0s6olmdlxr=w7NgV zF5;u*D zv{dWMci&3)_{--Vzir#Tl-y`4UUskb%lR<#iIQtnv;GbAYK|Ywx19B}FST>2M|*p) z(^N}zZ;+s8+`zjt)cLVY-E<(XcnI|<86{dN4XIq*Tnkq*+tKxU&ptNT>xHIr9^`1eepe5^a<^!c!Q~z>;RPrlN z%QLqpCu}2=`_~iaDTl+1NDovll7Gl*56rlXZTm4ujZYf-$>@^6LpR%p4_*_9+XZty zDN`xXlW4AR9`KrAs)2{?_Ug+O@#hcx$p zIFj~zRO;WsRW=t>2j`zKC9dD)u&0)Bp?DGSIvx%%FAy=z4UoY9Fn9fLfH>4^)HgB=zqJ2iI@1+;4WC`r=VL z4FOICrse*}-uU0M)eR|R4Y;9xV$yF?UNAS{QUwGdK2&=_08!#pO#vA1e=vaMR1^ZX zD%<_sy~_2E2ri5d6%hdeJl(u79zH%mAwq@Y{LfR{Kl|TxRR8-)+x*CHf5mZsGF89; zS2rM}phEIbOoQ>EqICXOO@sf`vVRov1q1rtHyICb^z#DCB5Kpbe_F)*tBLTW*{tl< zkPfCc*8%HJCyQTbLhe6TY=>qU|GqbwI?C9X=Y72L^LVB>l&CHlj)-^UT3(K!G>u-XS(|8*YO2LH!(+&tQ z76SgK3hv(-`)w`zbMw!4vGBSnAc+33j5Au=3L%|c5x|k-n{v%&4!qyc6?hHC-_N$d zj{5K0)_!TI!eGGM1Heo$N|C390gwy=M1X<+u^RuEF;X&C7M8ZaKUB;Yu(b`);emH5 z{>+dcQOdWSPr#B5!0w;#d`i(7a|Slz13RA_w_G^2%N51 z5)+RqXEjA4zWR{UI?>6H0UUHI9y-!4ly3d_dML0+&xlx^2#5oTt2HQj9aG zR7vhILChqc>QYPL7a}dlBPonyz~d|)l-W={&PT)Puw3e~s?Wbsxl&uh62Dcx-BIo1 zx7oG&Ds*W}Yz|M)xvO{6`-=|#dIm*d?yD%O$LA|-j8-HjtmHN*3!l&06Ebh($?E@bFl8w z3v@^LjP&ME?U8UIX+2C2=`ay{uKbBhP$x6RW9MUKvwYubmZE27ZG{mZ^Av2ZWb7cd zryT-`@9rm)78ACHwGj1Vcr~a}RoTu1kT!D3j?n;IY7`%8n&3l`QH|$OTI{rtALaD+P*)a;FJ$>h_EK z#Kd%OBL*$XQd&nII8^hACYuqFC-#Hw__tct(B<@^rWwdIAa^MN2Kc)x6YOyT(F9k&YoafaH=hO zrWv0W`)QBI?+YEDK2L3yUK1m(N3^2>#K!k1d%<7Mo=M8(VKbRHt23GVrvM|*VL=MR z{M&un*F!w9Z}87C$6r#3@(>$bfHGgcBVR|1qmQg^%2SK}(6EMR8C5(sRHwG#4(}}E zP-4(Zw;jR`SM937*XGsRrL&5taai0py}yysl6f(e>jI(8{JWzpfAx@pH!SLqBwVXy zy^19S3;&#a@Q|RD(y}3kPSaEC8P4_!2SQqqil1xvTNVewk}v=8Y8 z=9z~|HzZ!s(HExWujP<3_Qafxx=R}Xx95!@VhC=+8$z3o-+^0>UV>pw2BPWCCR!wM zhg{xdvo@-DWpC6;a^~%}Zov4P?aLZk)(y6{ZZ!#H8m(6f%;~v$L<$xXsOn~bJ@0*f z{=CX^%$)wF5`!EO?^s>dg5RoK@faw+d;dMT^>Kz3k@)#8=|<)5W7ivlseYx>1?r+I zEk^_vI9VdmVpb8HA;-usF7;Egk9nV|=O0LC$_!&QQmi;xl$;Cbq`h!7N-Ns9|DL4= za?a)pLs&-5((=x8d62tgtX7PejOw-UH5{V3 z#&UR*R8%(6fd2eEg(tf!8lj=jDEo-ga!d`oF3V=Y0mFXA-v8Ab0=K@|_DI(b_=oZ*`KDVwx)MGB{Y<>@E4X|5ATl{3_|xMUt7m*s7oveU^!Brn`NHD4Q4rE6RNn}r@<`1Mg^3V#(f&haspp|&;z;tWFG%#^uUuU z_*XrU`!x9Ghq#p!sB*H)iIQDOQKOxDe9YR2u5EPIZ zfq*n9FcbafHMzg)kNtDCYKQQEXeF#8|A8O<4sU=`qC?LFZ1GND`S@ItZ zk~2ZQ_2wrYMV=qA(Ko$^=U2nvwDlauxJiBciU<#t84R9<)m*<#Q<-wE3N`)2je-x>JSimzUBhFr1< z_0=~Vuir}n_Eyc;e7&3)WH=+`JL8d?P{kabGT(dP>|=ECV{7%TsXII`Oi0OjdIO(- z-d!HwTF$KVNe)So0hhWo$#|2E=d4I?JiSuUyZDTF;n;h39$jkN{xh>?&AHUWH>)nw z?*p?mZD7Oi2;S4~K4-h7JQX5V^y#s?1V8FEH#;EbGY?W2hj%3Zc+E|aai795LsabZ zfh6FMvx*Aa31+&*0W`$%m>HM=Y!n1iB_yN9)oe5vI&~g`QK6y246= zS^RcTGlkP|3ONS$K>?-w5gdd(uG&Me-fO+yB+J20|Dd2kmwj5aZ`1UxK^V^Gp;%%C zS%29QO0zmruf3Ydx@Hz%`qy;&)vzMXo|wCx8Uo#olg@!{YF}?3^*P|dt@G~=a%VAI za4A2=#mKy+uNBXN-EH}1NMgf7ajsgH%)Z6FQN|$o1Mar9TgKTfxWhD}QjZRc@ zek3KO##6PCO$6^0)A+hUZ=vqbS`;%MnXVX>ZzbC*^(rp1 zl)0<4m)Vi;o?Lzv8gG)g#|qKN&4E0s;7Fp0V%47F&tP?0y!Y)Jc?&}|#O8y}gD(@- z-77i6o={PQ5>WaUi*4`L-1j{aUIFLrY&3UA!)?=gp4DnuE5LLUXh)=D31+5Ci+HYX zfHBhY4;Vg?`j%sR-A`%#65xFgN5z-PT0u6Bvcvf`cZo@r+0|o>z`` zvw0vub+X6X9jC|1sk<9NLmEWwk*QcI!m6k>XC7eft;a%t^aE7~Q z3!UNvse!!x#j=H>Hu2!eR!h;_dI>Md3eB3jr1`CCF2z2>tgB*tP*$S*xMW~MFDX`9 zufoGUV4FF_`i3PnQrB;|nA6Uywx>}*UF~a_QfHSp!}C^wi&`~TA!m#pX*s4o>tRwa zNf>W0B-OF?$j=LqN$4Gb?ifdNR&nbqc6k{Vr1lQNBt&@`HbnH_O_R#p;*roM#9hgh zztVM|aM-i*Vlq?7I-P=v2%Z6fznNK#h_ngKGDSj*l*cpwz)0QH2;#45d?UP|w^!#c zJPQ!JJ@A)q6rrvc+9GH3!#}VvE?RAsh)& z7ptO2F*MHt)oDB+3Auw$bIS`Z+)uKSV~Y8zS%wTk>wKS!ey{xAd457%nkbdG3kf%B zkH^V#*S7+>r{UHa+zJXlyN(p*S84KABr$P3FVEFF-7bYe+aABNP1b4otQ_M)%@>$F@u0%ZHrh~9r za2joF|0Vm(v?n*&?Z_IJnfb-;J!NjVBY8lfYQ#HpCX3zyKhJ;hjLOxH+J{IMdA%qF zu6kFS7z(3vLKE)ieD9~ajePlH+Dnh9C$#SA8J#+wq36ZEVrGD@M>Cb&P)!nIH^J2^ z?a18|>U+z(=^E?qdg-+kRGTI_=Qi#o9;E;*Es2XUiqYK|>|?3SF2P^lqiix%;=}E6i5B zk)Eoi4S7fY$oJgcE2(?Q|6g-o0uN>PJzj5%t(|DGOo}kXtj3l#QdvTj5uve!8Ee*f zE83;Sl1O`6&|+UJiqawpWsoIQvKA8ZyU+8?$jo@(=>PrxKVKgo{q%6XP_Kw_nxbkwxlcy;%F7@v}&oR2?e&P;l#WREI znMd=i?q?*GW0mt8w;P^tis>Tfyc2Bdf4QV+u36C@p{vEm+7oZq?-%(zuj&n6LohS) z`{7n;;T^OaG8Q*;B6a&xgQDwu{Bq^mhH~j-?&)hXw!G+?_V}UO+`aer>@>DM#dlpY zgODvLQ%gELb5uLdwd};IVf9r!T&p&#{WujA6YNiJYBH+`coO-_VZ$PIqmxgr9a?ck z&bV0Uui>;rbaPWgw9??(g6BayXR60u&)l%fK>A&WF#1~9r)44c-@s#s& zqJb0gWL$AbQ#_vHfa{7P5W^AUIUI1uQ2@Hc|7;k_iHzsqoE(!J(iBgBW0M72m|=<5le|9u-X=0&rzCy2&x>j@HvI5Gz~$xCR`6px2vBZY0uu#idz#C!SgTbPMsP>#-y zws!vB6mjMd|NY=gn&M#*562xjXwgyV*{1Outy-q*SyR z3zkr~ZB9k{%Exz%&2}Stzfaa=)j6 zDGDSor9*NEFc$;+84lddVS$<8f1lF5DSqCP9Ei{b0ol=r6(sN|Mtgy@HB_`e#n#gc zZR5 zfz^h-v8OoirtCyJQ_-%@b`(!Bv{P7viC+TgBnHodFlV-|YtYCXA>f=TpqBt3{1q^A{LN~eqCka(84KS0Mk{xH@}*C}F=(y99i!f3X#Y7FP*Ag8klV@2cgth+5*J z;PW;Yze^BzR9w>8?jLuoW9HEpg)iqUkQG+Hll!jDvi;-7GYL;`zZaKCP|gvdcOTQa z`{ZVm#p-Z1v7ZGGyH`H_i$<;=_SyeJXLq9{7(%D458LWGKF z0y;pCd%Rvuaq;t^me#LAOX}t}I~pEVPz`_6l-=EFyZRv^!BZquYSVt%qiVw9i;la7 zT;U?#k@Ksp&=1vu)wpKQu1Nf~cecls!tcdvH|!79d8_hqUFu8;{f427LYE^?*gnnu z@vZj5L3_nrU1g}k=9QZaic1bQKWTkN=dZgNGZgUtobyT>^*4LcFK@TDSpRa(kskdo z<)uPZ_gYVOobcB@U+Y!q{s^@sW^aUtffwN~^4fWM2mFULKRnrQu_jeIsf%*Rh0HzZ zxIic6h{vv%St3d!*kuSBvHri58W4iJ;-g8UeY;e{TT&X@URW`@a zxp+9_Qrd|fcN~^$(CX{f)~>mQ-o$g{^_!B5xy>mt{S}qHd&zHSrcBqoxatd8->j?T zp!#Qrf#Zw)V||kbPY88as71t#wzdW6k%>Wioom*1A3i19ddMnO*s;pjG9tAmp*)34 zsK>M@L$5}-_j9+wHJzXj^RGqDl;R6>K4vepNx#f@YtFKz(+yW>aeH}2f8VpIf5HE(b)H9@7nDNbQwqSOl$(V=OA(`Iw8s^#wMp0#tyDxwYl&cEv$ zNi|RXWyKH7**UYO!%T|7B8waXM;n%fS^-wFMmYC9UiuFPvFbJ9kM@ zJD2ULx5v0=-Q8ool!)?4E!%eYbdf({Fu_8lAHVm@;ChNyjoSHZ0Vz&BwXM6h4T3Z= z;*Punx+FP@?hl2&BNqt5`)`M5%?s6_7dUT~P&yGHC|~9yUv%?{?eds|1(o$#SIU|z zbW1fd$yFIgN=o7jJk9QXl)}p5xMSXz4fx(K813kPw!vz}Qh|+6PvGA@3i>L3yI_Na z(MzZ4k&hxSF5p`i-XyELYVA70qn~$K4k~4;k@w zaHn@|iM?mvk@3_h?|_G9BBuQTjU1Dh^H*Nl>w)@So{&`CxvFn$wnfFQT-M%1Z;+{3 zn$Y~pQ}^4+*2gEb-%hXW*7MH|a9+ z`MrjOrbnYu8-4*A4Q#o{OIx*t5C3oo%-|B+3y{nQ)}1UHj`4) zK1r}IGVVN|cz6D;GX)>hLPg$hJJ)V4csQ24Icydx z6n;{MbaD1mbQ<>x)S>+kqc*O$4qIlZ_h3trEmxlK!Ts~?y}oEEt!X0%f{^u&BX&v~ zt}XoA^or!mUA}21YqzH<*3TIUUtt?4H25>%_R5A^x}7)W1=jq~+GlIuLX>%9T75~* z&k%R{NVFH%*;NZZ+lfVF$2_^c)AOsf{gut7XL)vFk}8HE+8K9~RA4 zK6%tp(dzN#)x=9p_m1q0kJ`Oiu`{wILukNYt-N~exw-jenhy^yzNRI)u+pK%c<9i_yp+F`F2REdhqJ3ezDDAdkl{+VU$@&s>BxzAp1SVLztrAb9iJ$s#bBS|$nUu?{T#@z*GmZ7v)6wUR5`*(z zqs%-WJXv+!^8)|Xqq-N5<8D6LHaqW$+$wV23CcD>{AtqCR**_8F39n-zTCY3`)t03OcUoo!=cjPD#aukFvT5KMiCD7c zN^NtHs)t5SZ=~J5)!@45u$V`R=&@bbZyC{auC7wo4=-$}Zf_4*q}!We@uqLsq)z6O zW6opI6-&kMuGczGXeMx#(A%{CNyjkhufz9tWVB}wW4`Tweid$v4@Mj+=&zf1?ug=ZY`(!## z!@0K+t<*h>7*YqJDoC}RLAcZ<2+hDt%dt>tbpW^ z!8-FuiHA8sFVk{Gg92hlUr6rT8nd#-Z^ag4%3wxHe|G5KN6yJPHOP&At6FFMbgN?2 z4LjHBWp|q^UW`tcXilESc`-m@o?gkkunOFS7uFYp59mk(K9D;ASX&25$Q=Msyusfg zR;aML!h{z-?Sl^pAoxIT5|j`qN(hv(AZnyv8jk;&y8{>#tak^n49;?Q088Mob2@=E zW{e8dN54e%N$AcnzzPw>O#|_|AUzZm@tpvos7=-a3>dCL2`hY0z%^x%*zy2s|b1EVT&~;KJXx4@Z_`u7zJdFa`t0f9=6G8r{Q_Q1GO}h zOo57#0OeYk+`J&%VS;}A5uHLlunZ$!2Cd?8lMQqSw2A*fBXG*PDbp$*DD4n6*4X?2 z&-cmMtO0$`KlSC0=oK>8{Y9@pl|5N{Wu)nwV1E9HUU4AGI=Bd6%njuIIZ1kjv+Pa8 z&&K6tnF75+@%EVXNRp9J&#!5>48A%W&E<*xtv_2v-_rcahDK_6eS@DY6m8Uo zddmZq^DhtieibV_KQcsXf05_^Gsdx`BWWc$d$9ka%AQD<+T)ueR#Ja-JO^Kmyb!zp zy)y3p&-RXIwEKYrG)0@*_@9IKYf6hwCwi-@4i4w$9!v}#tQN8v{r22~_OrKBOuoJ# zq(74u95nLcs4vLjRj|kAen4OMo54N9-9KY<#X9SMj&#PE_9w=*jrM&}EHjGu_$)%S z^3zv~UOlEWNW9`6APMk{vOhBuGss0|DoQ;w+>$J%3Rae zl)Q0ire0Uc=C1d{gGbfx2aJ;Hhg%vC-+rGL>^l$;@^(JguEfltj~a>1(MvKZ>FHG? zCrg`uH2VeyobR})o`2aUawW~;Q-0eEq0y0nini94I(5-`Un`am_yiuaanJ zcfLl!Lb)(9Z>9j%AdApg6-wr(ZqM>*tO}zqplWB$Z6t=#1*oQ328|h^^hMO8xt05w zzlC+X8mZ~-nU-1P@Qf1zaV%mq{k;#GH- z!epLI$=V$E_QGVo%r&(K+?N-+ac4@`UUQEtbmPgCsEu|f*UHWBjOOxvwm+BBNZpo2 z7VM0U%B8eWw`G&fnE)&N`#!DeBUNo7hXj|>f4I?oR5z;=TJk>*L=Jp&RP7)A0g&xA zkSS(UkU#XZ+b>v9;CC}sr%ZKUvTqLSDq(oVvv(1|=!(9j|mD%QPmgijNrVz5K8Z7s_7^D!g zt3FsBg#g|$7u|}UpA^MaRXW3k^3ZbI9dp5?sKqXn$Clf&&D|LQ)y|szto_mxVB(*1 z%(}g(wCa29mEKG0MlvNG*5NrZmw$G4*5$_BzMpTiaOnGDoBY6@{<=Cvi=v|4D$Q^8 za#g$g;ys$u;tl=%;x38gsh#WVMo1@6`D>#b2l-0Z!l>W@GyxF@potX-aqz;#L57Iq zaoy*KzkdITe7V757$aYm?8@}kX>RH|%P)0Y+MXw}CCwo1bY|)y;yN@$X$Vw6wjS=8 zBqrrH!j#)77?rzjF?B_)7FcH+8>69h#?M@3$ovyS;ZOo6+7E|xm_3vJCWp4TI@-_7ii{1D;KDePT8I;ef-woIT8R|J;3sBD?wgC78yoMVy zvgiOLxidp(V^fBEX8zVwnYC{6(I4b|=|?5}(J&VH5G?S51w7GYfd$MmS^YKo^`4pe z0gE!jay$A?=e1A+XGu`ezH}EUc7S8-iv_V`?28?-fIWmUc!N328x~XTtWPn2d;Jdi zmj|c$_3k>9Eq|q0)IVb=XOF`n|99$1*MshVZ4>*M<7>1huc~*%B|+FRVXa5=Fufq= zV1B~p-3e>rJNKad={hR@x$=P?5aHIv-#56TfOrA$o$#gaAJkGyR7Tf z5`5MMyx>oLnP1rBeKYu{*>>T9!##~n9iQHW!PJ{*DpY^XiHp{}!0nJZ$BIMf)(4?m zZG>(GpuGp;xJ}f+u$2JG){Tegg?0V!PeM3h)I4FsF}z38sUv<(k?n_f_heYWT>UuA z)jL!Hmskvz#Y9kf^&nas!oqoT)scF__)wWk#cc4wcvYiNp9xo4=11I?{DOHb>&ZM zE!aHL(wHLp*8kS>Gi&0bzZxp0xm`M!r`F}IO$WPGhkxSo8qahSCGw34H{-JtZ^pcg z%{W=KmhmgtY0l?+D=fDX$PO*kbk9u7FnDWU0dMW&76t3(KDH>3&E4&`E5udM%b~SU zstcj@9*owg@G-Qqa~cG#W81c*|UUfb2v7NnZ_DYl%tlKIT*)kQY52d%^XZ(*HHwcwwXC- z#^#d%7zcSiitS!(pr+KEi_fijd9|fdh4vPnD4iL_Bg1bdd1I4F>=kTyDPE@H`lid+ zbEV!g6*rz=!0s=dBZIr~{0jDT=_Z-f>(4J?`AWY;#VLeh17Aq;@kPG7MX=?j^eY<( znoFfxII#oN2i$u&)L^)yYrX_}7H^e6or zA+*u@C6|~H*KMJV35Ft~G*P9sY8&%xWc|zE8!Z|zAa$)iJUZA*@3KDk(|=dIpR%TM z&8IT|Y~|9vM=4*u8^3)Eh=)X^PV&C5#C0lufy!=8a;xc?Gu!0!;J=RGZwU|`pj{S{ zHYZJz?-#{`?T+=YFtpS`OAt4L+{Zkpkl>)4QE{={f`~T9 zYl%XGMO~3y%Ui#wv<~+_uJ-NfEEw#5*4ky$Q(1R3(#;j8sC^iC(i zG=0A5(wZIBwwYTqL-rk2F1}lG{5s#qg|Aj#(9?A1Yl_vpL)VNd>A#n(9>yM$(T%E_ufPhK0) zC>9(tcrl@%LPtq{Yo6-c%FD`oQXc}D?Gt&P;r_&PiF}EHs`?eh6^AR-q&~#4t92Y2KBYW#EDm1T^U8?82UY;^98M9v;fmV7Q9 z($a=^s}iDKr|n2q!#tL*s_`<7w@X%2dn{eeo3hvnon`Ks{K4{R`m*X>R<}Hoi_?{> zKU>}MN`9C=yLy?Gi)ZrFbjj+oRxTdNkJAOAQ%B3*^xg-IgU2lR406}pJSi&5WflT3 zyXx}Ej&(z^u^pds-RSpw)w1H= zs~t|!YlKT`=sep+JHK&two%h zwJ}^Fc7kt!#2lC-*ywg-cXx&ugIecJe9kBk;ez;}G#8c6m!8?$0E7X>kI>)DPYq)d zETqz4oC*5|LU{K7Mphd-E(WB&p#rfiIl#w9_Cwr!#!QHVtL*J&!{X$Qn*9dr`~d4t zhX2mhh5mQbbti+4zvHGcS+`c!o~)54l%euyziM=KKWg{+IqF|BEP@>xQY})pe;BxL zVHI2>)dkc_AtRql#HnS``h>ZJ!FA-v!=l z$T+;aH^cp|32B>jZIan>a~TVCRBDvWpB%x)&dXzmWjR>agZ90}ranTD;?^}?g+5r4 zaNccT?VT*>(AcSF>w%n|>jED&WgIqVO2^2FS-4hi9<1&GhvqD@8Ox!0vO*l=6pqj= zI5`7#4|2K#>K?}F4iUBk@yhv5?;{LDz7D^ODf=mQX((eA=Up&}F~ZO;lX>AgO%q;l z7X@p@F~HRcy6wX#KuTZe6S;-Nz?Vv(3{2|bdiX*GDl?t~0ZZX}SeXy%8Hvq7`WQw% z^4zbRVNi|ea{p&8h7q)|-V9?HS;fZ1DMD1vP~QnS#WBA!%v95XL!oJ4pB*S)5O7O? z45opf3u&_Ydgzj3i-|r;%E{Z?U0Gg!LiYq6V1(-FZ07?^YE6Neuf5a18_i;Xp*C|` zCUq5q1)Wy{ag;&F;Xvg$Jh&W!T(15&Q2B(z(Dk8$?8h`JoLc^OLztq;Iu2aO!(E04 zfQQ1h0Mmy5@BmDql)>6Gko_Anp8nkkrg-;*2Ug}FCKxgU1S|%QNBSQgfVQiT7l=R# z44WrCiYXdk!=~~`Xm}`v7KjG+=YD$=644(FVTu=DSR6>Z4j3Xb3M}M-1Wn7I+WSv9 z3SCcNf^Q5`n*EN5rs$#u5fO+v!{|HYC4~pxDu2*><)AV$XXDJncXwh|@EpMyh^M`~ zNbdvrQ`?-v8{7ga(kPbaE~}&s7UaB5O6jlAJG>+_EkR__>tKs!MTbS3XaXM{wrJN@ zS9XZ_eJvlgv#{f>N(;Dy_MWRUcS}X{&7~!(wj0(R@|68<)*tarGlDWd{bi$;-Mn)B zJfReaO*rvs_(G4+ci}vSTV+?Ro9|_^xS8Y->Mqcza|qKC)7q0RC&r7rJ8)EAId;J^ zqts()KtjQ^PcJ21pH<1S6+r76KRNc{Q6ZKA{JDzLqK9(}C>~P`2 z845=q{*bLW`N-rUS~FRfzVNP2fyBdoJX+4& z67nh)(F6HCWN&4+R69w}=ihL~vW~BJ5!bJmc6hMjdd$W-LK~#p?ulF5Me#h}E_bbG z&Kv)hPan@+srEnJr?OpT*S%{uy&YQ0^KC*6`Hw6Xn6*4?$ZaTKtJ>AT&pzau)BgPa zXN;PhG+tgQT$m`c`DNxDwPW*lScOK$eKy%q@A z_EWqtUhBL{sSy5NT(7VHk#B8Qq9R)qj$X(<{T=V8IjpH>U-VTuWe>f*?%ewqcf8zQ z-|HP22%dNE&db{QxoN?BOufX9bGIC9dH&+Zszw8SRr$YO^DOmDf4*@+NKog{*LW1q zXp)plZt)?S0>1Xmm-fr;vnyJjN4zx5eN}x;U07CfNwKxC3aX|o#%Gqu)j*-+HTUg> zm(H0${DYICLd%ZKi-;{iUW0dj=o7J8!7y^9g1}{n{04mVVEK+cO)3V?xB9;($uZKt zg1oKTfL($u*9(MTLnPGklH6FTRFcA21{x5ZYP<~O?CnZHNtxN&fe)Gk3UYvgz)AC> z!85W{IC^`s9$hSpnk;=n*gk_ zCuKJPg$UkbtsrtW)C$8;aC{>y5(Lr+76qk`boQhGSF3;MKcgXL{~5zU2ml@q!xV6Q zODwxQ(h?4(NAZRtqoIJ<;|WC136aHUj5!QOV+fTYjUoLQ-x$k&j5LNv=~H)7{Gir! zY`ty&ttsX}7)>GOb4XLjSL2&v**=k`2q-wM=wzB=4ujDYVsQi+sd#!p6oOD9N`8|q zOHrHV1JM&8EkHU9^S<$Yf?i`I9pEdE(xJG*f#^69Fn)+gAI5`&YS_RLQLue7^ZgP z0kOCwBY+f9cG9+dMUIve1gdTC13Mo_DE7wq zzKtI#!@_Mm&lSnnu#En6=N^ewbu`FJ{mz%N=f*PYNe>!|j$ozQ}mw{WIvQoWp+6<}B4g*JBx zCm1+hG=iO}HMgGPx*Kp0d)q0r^^0*#tt~TWH;R{>2@QmHoqTKKGU-gMwXu;+5tVS1 z?f)lK${E?YI)kXbQyAngnPX~$WKhc#TKdI8rq&WNc~hgx@hcXlcj(~i#S76mAax~P z`Hjvyy-;L$I|Mr(c-#a8N{_!9E}LY*vyazHrd)yT)j|?O#mftmUyy;(T$cbjoI@jpAH_l0#m7p( zHJuUv9Ri~SN{np@loUY;VY&bl^2VMtdsB)xiVV7`ql=Q)cLP}0Vti@?rUUVD$h4p4 zO7k>z2VrbcNau7Y4z@n7-X?llD1>JNXunYVYt;i8)W;3Xn*w+bw1JYsNJCQq@X9D- zkusK9#vx@Kvy4Z|cxIV^lnKl-5h)XyWksZ{$SjkPGKpDMg3B-+FbM$`ipRDKnD{hd z9U`y1DN0_CW=7+c*FljPL?^#>n>34L#>zf9+lW);lb+M1O(91E&Du1mMO~4#PiZ zf~jCE^n)`78sO_8+@kA+q- zTOAJaUS+EzK+d0Rb&8NbENh*j0(4YhuLF^{IO?D@CaljX5up8@wN6n1@~dU7Q-Y$Z zvDGPpxHBAec*sMS?OjEVc?0?(=xUbzIVd{}OC6A5psQJyItAdE%HC(N<3S0US)Kzf zm{2ZiwmJeNmax?+LUhYkrvz!RSnI%TGDjOg1If{65D$=}4RA>Wcq1!3JPz{PWvx?G zng9oQ=(?KyIp_+UwGK~&uD{vpNYDnvT1Qaega^9Jfdc|?Zsd5E2xatUg-7DVJqS7v z-P*H1$3ZV3wJj7XkL@|mwSXZJCxAzS)WEFog6s+GYXPGKZn`+0!$Svqws(n8pjoy$ zPCf)uba2olh}g%$b3htw$laU;9*C>PL6cZu#lYS_kV^q_ac6y3QIP{zSm4jc@vfpG zXC02fu|BXk5+qNuwt-iGL{YXnJm-d1%sJX15IOiMNQMMO z{AKA24)9nGoPynt9alIU2?`U;@-CLd$#=0}80>9;ZGnToU`Zq>-Y_dXB?S(i4KCw3 z+f~9t$0W9QK^$9-Iub`64y1!&e-1YRUjtOY!C$}<@r~1L;cHaRu&qq2nS;8#uu0*xLZk=o~bO z0|Q~l9|+RQ!Jk3&G$@cXOB*09H3!cDZoC|OH+T+;-OTbX;07Er1IT#IL032emV=%_ zaxCcDiVYqJImuRs2g6{eNnjqrvB!hwIM)Y3k%O-AB#@ActqmNQA`bik7RE6j1SJl+ z1UNZ!&?MN&IQSYK54a~=yMWZ78wC~|;DIc_K3*V3Kqg?U&w)*a16No)C!VoHPPr2- zbdEM~3LJO=*^!{zIM#N-e$I(20>|D1LODR@Xl(C-!Ew|PIQTeN9uwe2k&`CDq;tYk z;-n#h0w>)A+U39j0g5Wl)@NW)#xX|(Kt&wS5jgQeAacmcAczA8zXF^Sa+zku6_HbZ z!4o-UGdvN?v5xRWz!5m$0i<%&5jglg2z@bL9{2VHp%W>dAe;!$(ST68@F7Ruga%@B z0?Y%Yn7lsKfrbKDgZ~HG8&wQRVV#zeqLu<)S4*3K( - + Solr Version Control System @@ -199,7 +199,7 @@ document.write("Last Published: " + document.lastModified); |alternative credits +-->
-Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo +Book: Solr 1.4 Enterprise Search Server - logoLucidWorks for Solr Certified Distribution Reference Guide - logo
+ 4.0.0 + + org.apache.solr + solr-parent + @version@ + ../../pom.xml + + org.apache.solr + solr-test-framework + jar + Apache Solr Test Framework + Apache Solr Test Framework + + solr/src/test-framework + ../../build + + + + ${project.groupId} + solr-core + ${project.version} + + + org.apache.lucene + lucene-test-framework + ${project.version} + + + junit + junit + + + + ${build-directory} + ${build-directory}/classes/test-framework + . + + + . + + **/*.java + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + diff --git a/solr/build.xml b/solr/build.xml index 5821f9953e6..bc1999e367c 100644 --- a/solr/build.xml +++ b/solr/build.xml @@ -176,6 +176,24 @@ + + + + + + + + + + + + + + + @@ -221,7 +239,7 @@ failonerror="true"/> - + @@ -320,13 +338,18 @@ - + + + + + + @@ -337,10 +360,19 @@ + + + + + + + + depends="compileTestFramework"> + depends="dist-solrj, dist-jar, dist-test-framework, dist-contrib, dist-war" /> + + + + + + + + + + + + @@ -1054,6 +1102,14 @@ + + + + + + + diff --git a/solr/contrib/analysis-extras/build.xml b/solr/contrib/analysis-extras/build.xml index 1b135e3c4af..9cc5aa217bc 100644 --- a/solr/contrib/analysis-extras/build.xml +++ b/solr/contrib/analysis-extras/build.xml @@ -72,7 +72,7 @@ - + @@ -92,7 +92,7 @@ - + diff --git a/solr/contrib/clustering/build.xml b/solr/contrib/clustering/build.xml index 7090ca710bc..aee297e3b8f 100644 --- a/solr/contrib/clustering/build.xml +++ b/solr/contrib/clustering/build.xml @@ -41,8 +41,8 @@ - - + + @@ -64,7 +64,7 @@ - + diff --git a/solr/contrib/dataimporthandler/build.xml b/solr/contrib/dataimporthandler/build.xml index 5b9ddc18dec..bd6ea50a2e2 100644 --- a/solr/contrib/dataimporthandler/build.xml +++ b/solr/contrib/dataimporthandler/build.xml @@ -55,7 +55,7 @@ - + @@ -67,7 +67,7 @@ - + @@ -89,7 +89,7 @@ - + diff --git a/solr/contrib/extraction/build.xml b/solr/contrib/extraction/build.xml index de7542d54b4..01aa60e7485 100644 --- a/solr/contrib/extraction/build.xml +++ b/solr/contrib/extraction/build.xml @@ -39,7 +39,7 @@ - + @@ -51,7 +51,7 @@ - + diff --git a/solr/contrib/uima/build.xml b/solr/contrib/uima/build.xml index 34dbefec748..631f8a8c413 100644 --- a/solr/contrib/uima/build.xml +++ b/solr/contrib/uima/build.xml @@ -40,7 +40,7 @@ - + @@ -52,7 +52,7 @@ - + diff --git a/solr/site/features.html b/solr/site/features.html index 6874449de23..f64d00e1d4a 100755 --- a/solr/site/features.html +++ b/solr/site/features.html @@ -130,6 +130,9 @@ document.write("Last Published: " + document.lastModified);
SolrJ Javadoc
+
+Test Framework Javadoc +