Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8166: Require merge instances to be consumed in the thread that created them.

This commit is contained in:
Adrien Grand 2019-03-15 14:16:37 +01:00
parent cf828163bd
commit 577bef53dd
29 changed files with 487 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene70/Lucene70NormsProducer.java
View File

@ -91,7 +91,7 @@ final class Lucene70NormsProducer extends NormsProducer implements Cloneable {
}
@Override
public NormsProducer getMergeInstance() throws IOException {
public NormsProducer getMergeInstance() {
Lucene70NormsProducer clone;
try {
clone = (Lucene70NormsProducer) super.clone();

4
lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
View File

@ -80,7 +80,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
static final int VERSION_CURRENT = VERSION_START;
// clone for merge: when merging we don't do any instances.put()s
DirectDocValuesProducer(DirectDocValuesProducer original) throws IOException {
DirectDocValuesProducer(DirectDocValuesProducer original) {
assert Thread.holdsLock(original);
numerics.putAll(original.numerics);
binaries.putAll(original.binaries);
@ -606,7 +606,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
@Override
public synchronized DocValuesProducer getMergeInstance() throws IOException {
public synchronized DocValuesProducer getMergeInstance() {
return new DirectDocValuesProducer(this);
}

5
lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
View File

@ -74,10 +74,11 @@ public abstract class DocValuesProducer implements Closeable, Accountable {
public abstract void checkIntegrity() throws IOException;
/**
* Returns an instance optimized for merging.
* Returns an instance optimized for merging. This instance may only be
* consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
public DocValuesProducer getMergeInstance() throws IOException {
public DocValuesProducer getMergeInstance() {
return this;
}
}

5
lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
View File

@ -48,10 +48,11 @@ public abstract class FieldsProducer extends Fields implements Closeable, Accoun
public abstract void checkIntegrity() throws IOException;
/**
* Returns an instance optimized for merging.
* Returns an instance optimized for merging. This instance may only be
* consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
public FieldsProducer getMergeInstance() throws IOException {
public FieldsProducer getMergeInstance() {
return this;
}
}

5
lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
View File

@ -49,10 +49,11 @@ public abstract class NormsProducer implements Closeable, Accountable {
public abstract void checkIntegrity() throws IOException;
/**
* Returns an instance optimized for merging.
* Returns an instance optimized for merging. This instance may only be used
* from the thread that acquires it.
* <p>
* The default implementation returns {@code this} */
public NormsProducer getMergeInstance() throws IOException {
public NormsProducer getMergeInstance() {
return this;
}
}

5
lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java
View File

@ -45,10 +45,11 @@ public abstract class PointsReader implements Closeable, Accountable {
public abstract PointValues getValues(String field) throws IOException;
/**
* Returns an instance optimized for merging.
* Returns an instance optimized for merging. This instance may only be used
* in the thread that acquires it.
* <p>
* The default implementation returns {@code this} */
public PointsReader getMergeInstance() throws IOException {
public PointsReader getMergeInstance() {
return this;
}
}

4
lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
View File

@ -52,10 +52,10 @@ public abstract class StoredFieldsReader implements Cloneable, Closeable, Accoun
public abstract void checkIntegrity() throws IOException;
/**
* Returns an instance optimized for merging.
* Returns an instance optimized for merging. This instance may not be cloned.
* <p>
* The default implementation returns {@code this} */
public StoredFieldsReader getMergeInstance() throws IOException {
public StoredFieldsReader getMergeInstance() {
return this;
}
}

5
lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
View File

@ -57,10 +57,11 @@ public abstract class TermVectorsReader implements Cloneable, Closeable, Account
public abstract TermVectorsReader clone();
/**
* Returns an instance optimized for merging.
* Returns an instance optimized for merging. This instance may only be
* consumed in the thread that called {@link #getMergeInstance()}.
* <p>
* The default implementation returns {@code this} */
public TermVectorsReader getMergeInstance() throws IOException {
public TermVectorsReader getMergeInstance() {
return this;
}
}

85
lucene/core/src/java/org/apache/lucene/codecs/lucene80/Lucene80NormsProducer.java
View File

@ -92,7 +92,7 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
@Override
public NormsProducer getMergeInstance() throws IOException {
public NormsProducer getMergeInstance() {
Lucene80NormsProducer clone;
try {
clone = (Lucene80NormsProducer) super.clone();
@ -233,18 +233,79 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
private IndexInput getDisiInput(FieldInfo field, NormsEntry entry) throws IOException {
IndexInput slice = null;
if (merging) {
slice = disiInputs.get(field.number);
}
if (slice == null) {
slice = IndexedDISI.createBlockSlice(
if (merging == false) {
return IndexedDISI.createBlockSlice(
data, "docs", entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.jumpTableEntryCount);
if (merging) {
disiInputs.put(field.number, slice);
}
}
return slice;
IndexInput in = disiInputs.get(field.number);
if (in == null) {
in = IndexedDISI.createBlockSlice(
data, "docs", entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.jumpTableEntryCount);
disiInputs.put(field.number, in);
}
final IndexInput inF = in; // same as in but final
// Wrap so that reads can be interleaved from the same thread if two
// norms instances are pulled and consumed in parallel. Merging usually
// doesn't need this feature but CheckIndex might, plus we need merge
// instances to behave well and not be trappy.
return new IndexInput("docs") {
long offset = 0;
@Override
public void readBytes(byte[] b, int off, int len) throws IOException {
inF.seek(offset);
offset += len;
inF.readBytes(b, off, len);
}
@Override
public byte readByte() throws IOException {
throw new UnsupportedOperationException("Unused by IndexedDISI");
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
throw new UnsupportedOperationException("Unused by IndexedDISI");
}
@Override
public short readShort() throws IOException {
inF.seek(offset);
offset += Short.BYTES;
return inF.readShort();
}
@Override
public long readLong() throws IOException {
inF.seek(offset);
offset += Long.BYTES;
return inF.readLong();
}
@Override
public void seek(long pos) throws IOException {
offset = pos;
}
@Override
public long length() {
throw new UnsupportedOperationException("Unused by IndexedDISI");
}
@Override
public long getFilePointer() {
return offset;
}
@Override
public void close() throws IOException {
throw new UnsupportedOperationException("Unused by IndexedDISI");
}
};
}
private RandomAccessInput getDisiJumpTable(FieldInfo field, NormsEntry entry) throws IOException {
@ -327,7 +388,7 @@ final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
}
};
}
final RandomAccessInput slice = data.randomAccessSlice(entry.normsOffset, entry.numDocsWithField * (long) entry.bytesPerNorm);
final RandomAccessInput slice = getDataInput(field, entry);
switch (entry.bytesPerNorm) {
case 1:
return new SparseNormsIterator(disi) {

4
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
View File

@ -261,7 +261,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
private final Map<String,DocValuesProducer> formats = new HashMap<>();
// clone for merge
FieldsReader(FieldsReader other) throws IOException {
FieldsReader(FieldsReader other) {
Map<DocValuesProducer,DocValuesProducer> oldToNew = new IdentityHashMap<>();
// First clone all formats
for(Map.Entry<String,DocValuesProducer> ent : other.formats.entrySet()) {
@ -368,7 +368,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
}
@Override
public DocValuesProducer getMergeInstance() throws IOException {
public DocValuesProducer getMergeInstance() {
return new FieldsReader(this);
}

4
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
View File

@ -247,7 +247,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
private final String segment;
// clone for merge
FieldsReader(FieldsReader other) throws IOException {
FieldsReader(FieldsReader other) {
Map<FieldsProducer,FieldsProducer> oldToNew = new IdentityHashMap<>();
// First clone all formats
for(Map.Entry<String,FieldsProducer> ent : other.formats.entrySet()) {
@ -346,7 +346,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
}
@Override
public FieldsProducer getMergeInstance() throws IOException {
public FieldsProducer getMergeInstance() {
return new FieldsReader(this);
}

29
lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.codecs.lucene50;
/**
* Test the merge instance of the Lucene50 stored fields format.
*/
public class TestLucene50StoredFieldsFormatMergeInstance extends TestLucene50StoredFieldsFormat {
@Override
protected boolean shouldTestMergeInstance() {
return true;
}
}

29
lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestLucene80NormsFormatMergeInstance.java Normal file
View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.codecs.lucene80;
/**
* Test the merge instance of the Lucene80 norms format.
*/
public class TestLucene80NormsFormatMergeInstance extends TestLucene80NormsFormat {
@Override
protected boolean shouldTestMergeInstance() {
return true;
}
}

2
lucene/core/src/test/org/apache/lucene/index/TestMultiTermsEnum.java
View File

@ -222,7 +222,7 @@ public class TestMultiTermsEnum extends LuceneTestCase {
}
@Override
public FieldsProducer getMergeInstance() throws IOException {
public FieldsProducer getMergeInstance() {
return create(delegate.getMergeInstance(), newFieldInfo);
}

2
lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
View File

@ -133,7 +133,7 @@ final class CompletionFieldsProducer extends FieldsProducer {
}
@Override
public FieldsProducer getMergeInstance() throws IOException {
public FieldsProducer getMergeInstance() {
return new CompletionFieldsProducer(delegateFieldsProducer, readers);
}

8
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
View File

@ -33,6 +33,14 @@ import org.apache.lucene.util.TestUtil;
*/
public class AssertingCodec extends FilterCodec {
static void assertThread(String object, Thread creationThread) {
if (creationThread != Thread.currentThread()) {
throw new AssertionError(object + " are only supposed to be consumed in "
+ "the thread in which they have been acquired. But was acquired in "
+ creationThread + " and consumed in " + Thread.currentThread() + ".");
}
}
private final PostingsFormat postings = new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {

27
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
View File

@ -62,7 +62,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
assert state.fieldInfos.hasDocValues();
DocValuesProducer producer = in.fieldsProducer(state);
assert producer != null;
return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc());
return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc(), false);
}
static class AssertingDocValuesConsumer extends DocValuesConsumer {
@ -219,10 +219,14 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
static class AssertingDocValuesProducer extends DocValuesProducer {
private final DocValuesProducer in;
private final int maxDoc;
private final boolean merging;
private final Thread creationThread;
AssertingDocValuesProducer(DocValuesProducer in, int maxDoc) {
AssertingDocValuesProducer(DocValuesProducer in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
this.merging = merging;
this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@ -231,6 +235,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
if (merging) {
AssertingCodec.assertThread("DocValuesProducer", creationThread);
}
assert field.getDocValuesType() == DocValuesType.NUMERIC;
NumericDocValues values = in.getNumeric(field);
assert values != null;
@ -239,6 +246,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
if (merging) {
AssertingCodec.assertThread("DocValuesProducer", creationThread);
}
assert field.getDocValuesType() == DocValuesType.BINARY;
BinaryDocValues values = in.getBinary(field);
assert values != null;
@ -247,6 +257,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
if (merging) {
AssertingCodec.assertThread("DocValuesProducer", creationThread);
}
assert field.getDocValuesType() == DocValuesType.SORTED;
SortedDocValues values = in.getSorted(field);
assert values != null;
@ -255,6 +268,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
if (merging) {
AssertingCodec.assertThread("DocValuesProducer", creationThread);
}
assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;
SortedNumericDocValues values = in.getSortedNumeric(field);
assert values != null;
@ -263,6 +279,9 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
if (merging) {
AssertingCodec.assertThread("DocValuesProducer", creationThread);
}
assert field.getDocValuesType() == DocValuesType.SORTED_SET;
SortedSetDocValues values = in.getSortedSet(field);
assert values != null;
@ -295,8 +314,8 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
}
@Override
public DocValuesProducer getMergeInstance() throws IOException {
return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc);
public DocValuesProducer getMergeInstance() {
return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc, true);
}
@Override

15
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
View File

@ -50,7 +50,7 @@ public class AssertingNormsFormat extends NormsFormat {
assert state.fieldInfos.hasNorms();
NormsProducer producer = in.normsProducer(state);
assert producer != null;
return new AssertingNormsProducer(producer, state.segmentInfo.maxDoc());
return new AssertingNormsProducer(producer, state.segmentInfo.maxDoc(), false);
}
static class AssertingNormsConsumer extends NormsConsumer {
@ -88,10 +88,14 @@ public class AssertingNormsFormat extends NormsFormat {
static class AssertingNormsProducer extends NormsProducer {
private final NormsProducer in;
private final int maxDoc;
private final boolean merging;
private final Thread creationThread;
AssertingNormsProducer(NormsProducer in, int maxDoc) {
AssertingNormsProducer(NormsProducer in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
this.merging = merging;
this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@ -100,6 +104,9 @@ public class AssertingNormsFormat extends NormsFormat {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
if (merging) {
AssertingCodec.assertThread("NormsProducer", creationThread);
}
assert field.hasNorms();
NumericDocValues values = in.getNorms(field);
assert values != null;
@ -132,8 +139,8 @@ public class AssertingNormsFormat extends NormsFormat {
}
@Override
public NormsProducer getMergeInstance() throws IOException {
return new AssertingNormsProducer(in.getMergeInstance(), maxDoc);
public NormsProducer getMergeInstance() {
return new AssertingNormsProducer(in.getMergeInstance(), maxDoc, true);
}
@Override

15
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java
View File

@ -60,17 +60,21 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public PointsReader fieldsReader(SegmentReadState state) throws IOException {
return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state));
return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state), false);
}
static class AssertingPointsReader extends PointsReader {
private final PointsReader in;
private final int maxDoc;
private final boolean merging;
private final Thread creationThread;
AssertingPointsReader(int maxDoc, PointsReader in) {
AssertingPointsReader(int maxDoc, PointsReader in, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
this.merging = merging;
this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@ -85,6 +89,9 @@ public final class AssertingPointsFormat extends PointsFormat {
@Override
public PointValues getValues(String field) throws IOException {
if (merging) {
AssertingCodec.assertThread("PointsReader", creationThread);
}
PointValues values = this.in.getValues(field);
if (values == null) {
return null;
@ -112,8 +119,8 @@ public final class AssertingPointsFormat extends PointsFormat {
}
@Override
public PointsReader getMergeInstance() throws IOException {
return new AssertingPointsReader(maxDoc, in.getMergeInstance());
public PointsReader getMergeInstance() {
return new AssertingPointsReader(maxDoc, in.getMergeInstance(), true);
}
@Override

2
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
View File

@ -114,7 +114,7 @@ public final class AssertingPostingsFormat extends PostingsFormat {
}
@Override
public FieldsProducer getMergeInstance() throws IOException {
public FieldsProducer getMergeInstance() {
return new AssertingFieldsProducer(in.getMergeInstance());
}

16
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
View File

@ -40,7 +40,7 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc());
return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc(), false);
}
@Override
@ -51,10 +51,14 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
static class AssertingStoredFieldsReader extends StoredFieldsReader {
private final StoredFieldsReader in;
private final int maxDoc;
private final boolean merging;
private final Thread creationThread;
AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc) {
AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
this.merging = merging;
this.creationThread = Thread.currentThread();
// do a few simple checks on init
assert toString() != null;
assert ramBytesUsed() >= 0;
@ -69,13 +73,15 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
AssertingCodec.assertThread("StoredFieldsReader", creationThread);
assert n >= 0 && n < maxDoc;
in.visitDocument(n, visitor);
}
@Override
public StoredFieldsReader clone() {
return new AssertingStoredFieldsReader(in.clone(), maxDoc);
assert merging == false : "Merge instances do not support cloning";
return new AssertingStoredFieldsReader(in.clone(), maxDoc, false);
}
@Override
@ -98,8 +104,8 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
}
@Override
public StoredFieldsReader getMergeInstance() throws IOException {
return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc);
public StoredFieldsReader getMergeInstance() {
return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc, true);
}
@Override

2
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
View File

@ -97,7 +97,7 @@ public class AssertingTermVectorsFormat extends TermVectorsFormat {
}
@Override
public TermVectorsReader getMergeInstance() throws IOException {
public TermVectorsReader getMergeInstance() {
return new AssertingTermVectorsReader(in.getMergeInstance());
}

11
lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
View File

@ -1028,7 +1028,7 @@ public class AssertingLeafReader extends FilterLeafReader {
/** Wraps a SortedSetDocValues but with additional asserts */
public static class AssertingPointValues extends PointValues {
private final Thread creationThread = Thread.currentThread();
private final PointValues in;
/** Sole constructor. */
@ -1048,11 +1048,13 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public void intersect(IntersectVisitor visitor) throws IOException {
assertThread("Points", creationThread);
in.intersect(new AssertingIntersectVisitor(in.getNumDataDimensions(), in.getNumIndexDimensions(), in.getBytesPerDimension(), visitor));
}
@Override
public long estimatePointCount(IntersectVisitor visitor) {
assertThread("Points", creationThread);
long cost = in.estimatePointCount(visitor);
assert cost >= 0;
return cost;
@ -1060,36 +1062,43 @@ public class AssertingLeafReader extends FilterLeafReader {
@Override
public byte[] getMinPackedValue() throws IOException {
assertThread("Points", creationThread);
return Objects.requireNonNull(in.getMinPackedValue());
}
@Override
public byte[] getMaxPackedValue() throws IOException {
assertThread("Points", creationThread);
return Objects.requireNonNull(in.getMaxPackedValue());
}
@Override
public int getNumDataDimensions() throws IOException {
assertThread("Points", creationThread);
return in.getNumDataDimensions();
}
@Override
public int getNumIndexDimensions() throws IOException {
assertThread("Points", creationThread);
return in.getNumIndexDimensions();
}
@Override
public int getBytesPerDimension() throws IOException {
assertThread("Points", creationThread);
return in.getBytesPerDimension();
}
@Override
public long size() {
assertThread("Points", creationThread);
return in.size();
}
@Override
public int getDocCount() {
assertThread("Points", creationThread);
return in.getDocCount();
}

26
lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@ -131,9 +132,15 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
queue.addAll(map.values());
v = 2L * map.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
} else {
v = super.accumulateObject(o, shallowSize, fieldValues, queue);
List<Object> references = new ArrayList<>();
v = super.accumulateObject(o, shallowSize, fieldValues, references);
for (Object r : references) {
// AssertingCodec adds Thread references to make sure objects are consumed in the right thread
if (r instanceof Thread == false) {
queue.add(r);
}
}
}
// System.out.println(o.getClass() + "=" + v);
return v;
}
@ -698,4 +705,19 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
Rethrow.rethrow(e);
}
/**
* Returns {@code false} if only the regular fields reader should be tested,
* and {@code true} if only the merge instance should be tested.
*/
protected boolean shouldTestMergeInstance() {
return false;
}
protected final DirectoryReader maybeWrapWithMergingReader(DirectoryReader r) throws IOException {
if (shouldTestMergeInstance()) {
r = new MergingDirectoryReaderWrapper(r);
}
return r;
}
}

77
lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
View File

@ -37,6 +37,7 @@ import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@ -491,14 +492,14 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
writer.commit();
// compare
DirectoryReader ir = DirectoryReader.open(dir);
DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
checkNormsVsDocValues(ir);
ir.close();
writer.forceMerge(1);
// compare again
ir = DirectoryReader.open(dir);
ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
checkNormsVsDocValues(ir);
writer.close();
@ -605,7 +606,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
w.deleteDocuments(new Term("id", ""+id));
}
w.forceMerge(1);
IndexReader r = w.getReader();
IndexReader r = maybeWrapWithMergingReader(w.getReader());
assertFalse(r.hasDeletions());
// Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content". They should not
@ -679,7 +680,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
}
}
DirectoryReader reader = writer.getReader();
DirectoryReader reader = maybeWrapWithMergingReader(writer.getReader());
writer.close();
final int numThreads = TestUtil.nextInt(random(), 3, 30);
@ -711,4 +712,72 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
reader.close();
dir.close();
}
public void testIndependantIterators() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
conf.setSimilarity(sim);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
Document doc = new Document();
Field indexedField = new TextField("indexed", "a", Field.Store.NO);
doc.add(indexedField);
for (int i = 0; i < 3; ++i) {
writer.addDocument(doc);
}
writer.forceMerge(1);
LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
NumericDocValues n1 = r.getNormValues("indexed");
NumericDocValues n2 = r.getNormValues("indexed");
assertEquals(0, n1.nextDoc());
assertEquals(42, n1.longValue());
assertEquals(1, n1.nextDoc());
assertEquals(10, n1.longValue());
assertEquals(0, n2.nextDoc());
assertEquals(42, n2.longValue());
assertEquals(1, n2.nextDoc());
assertEquals(10, n2.longValue());
assertEquals(2, n2.nextDoc());
assertEquals(20, n2.longValue());
assertEquals(2, n1.nextDoc());
assertEquals(20, n1.longValue());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
IOUtils.close(r, writer, dir);
}
public void testIndependantSparseIterators() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
conf.setSimilarity(sim);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
Document doc = new Document();
Field indexedField = new TextField("indexed", "a", Field.Store.NO);
doc.add(indexedField);
Document emptyDoc = new Document();
for (int i = 0; i < 3; ++i) {
writer.addDocument(doc);
writer.addDocument(emptyDoc);
}
writer.forceMerge(1);
LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
NumericDocValues n1 = r.getNormValues("indexed");
NumericDocValues n2 = r.getNormValues("indexed");
assertEquals(0, n1.nextDoc());
assertEquals(42, n1.longValue());
assertEquals(2, n1.nextDoc());
assertEquals(10, n1.longValue());
assertEquals(0, n2.nextDoc());
assertEquals(42, n2.longValue());
assertEquals(2, n2.nextDoc());
assertEquals(10, n2.longValue());
assertEquals(4, n2.nextDoc());
assertEquals(20, n2.longValue());
assertEquals(4, n1.nextDoc());
assertEquals(20, n1.longValue());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
IOUtils.close(r, writer, dir);
}
}

24
lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
View File

@ -140,7 +140,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
String[] idsList = docs.keySet().toArray(new String[docs.size()]);
for(int x=0;x<2;x++) {
IndexReader r = w.getReader();
DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
IndexSearcher s = newSearcher(r);
if (VERBOSE) {
@ -181,7 +181,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(newField("aaa", "a b c", customType));
doc.add(newField("zzz", "1 2 3", customType));
w.addDocument(doc);
IndexReader r = w.getReader();
IndexReader r = maybeWrapWithMergingReader(w.getReader());
Document doc2 = r.document(0);
Iterator<IndexableField> it = doc2.getFields().iterator();
assertTrue(it.hasNext());
@ -280,7 +280,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(new NumericDocValuesField("id", id));
w.addDocument(doc);
}
final DirectoryReader r = w.getReader();
final DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
w.close();
assertEquals(numDocs, r.numDocs());
@ -309,7 +309,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
doc.add(new Field("field", "value", onlyStored));
doc.add(new StringField("field2", "value", Field.Store.YES));
w.addDocument(doc);
IndexReader r = w.getReader();
IndexReader r = maybeWrapWithMergingReader(w.getReader());
w.close();
assertEquals(IndexOptions.NONE, r.document(0).getField("field").fieldType().indexOptions());
assertNotNull(r.document(0).getField("field2").fieldType().indexOptions());
@ -352,7 +352,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
final DirectoryReader reader = DirectoryReader.open(dir);
final DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final int docID = random().nextInt(100);
for (Field fld : fields) {
String fldName = fld.name();
@ -383,7 +383,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.addDocument(emptyDoc);
}
iw.commit();
final DirectoryReader rd = DirectoryReader.open(dir);
final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
for (int i = 0; i < numDocs; ++i) {
final Document doc = rd.document(i);
assertNotNull(doc);
@ -412,7 +412,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
final DirectoryReader rd = DirectoryReader.open(dir);
final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final IndexSearcher searcher = new IndexSearcher(rd);
final int concurrentReads = atLeast(5);
final int readsPerThread = atLeast(50);
@ -545,7 +545,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.commit();
final DirectoryReader ir = DirectoryReader.open(dir);
final DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
assertTrue(ir.numDocs() > 0);
int numDocs = 0;
for (int i = 0; i < ir.maxDoc(); ++i) {
@ -649,7 +649,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
w.commit();
w.close();
DirectoryReader reader = new DummyFilterDirectoryReader(DirectoryReader.open(dir));
DirectoryReader reader = new DummyFilterDirectoryReader(maybeWrapWithMergingReader(DirectoryReader.open(dir)));
Directory dir2 = newDirectory();
w = new RandomIndexWriter(random(), dir2);
@ -657,7 +657,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
reader.close();
dir.close();
reader = w.getReader();
reader = maybeWrapWithMergingReader(w.getReader());
for (int i = 0; i < reader.maxDoc(); ++i) {
final Document doc = reader.document(i);
final int id = doc.getField("id").numericValue().intValue();
@ -728,7 +728,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
}
iw.commit();
iw.forceMerge(1); // look at what happens when big docs are merged
final DirectoryReader rd = DirectoryReader.open(dir);
final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
final IndexSearcher searcher = new IndexSearcher(rd);
for (int i = 0; i < numDocs; ++i) {
final Query query = new TermQuery(new Term("id", "" + i));
@ -788,7 +788,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormat
iw.addDocument(doc);
}
DirectoryReader reader = DirectoryReader.open(iw);
DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(iw));
// mix up fields explicitly
if (random().nextBoolean()) {
reader = new MismatchedDirectoryReader(reader, random());

75
lucene/test-framework/src/java/org/apache/lucene/index/MergingCodecReader.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.util.CloseableThreadLocal;
/**
* {@link CodecReader} wrapper that performs all reads using the merging
* instance of the index formats.
*/
public class MergingCodecReader extends FilterCodecReader {
private final CloseableThreadLocal<StoredFieldsReader> fieldsReader = new CloseableThreadLocal<StoredFieldsReader>() {
@Override
protected StoredFieldsReader initialValue() {
return in.getFieldsReader().getMergeInstance();
}
};
private final CloseableThreadLocal<NormsProducer> normsReader = new CloseableThreadLocal<NormsProducer>() {
@Override
protected NormsProducer initialValue() {
NormsProducer norms = in.getNormsReader();
if (norms == null) {
return null;
} else {
return norms.getMergeInstance();
}
}
};
// TODO: other formats too
/** Wrap the given instance. */
public MergingCodecReader(CodecReader in) {
super(in);
}
@Override
public StoredFieldsReader getFieldsReader() {
return fieldsReader.get();
}
@Override
public NormsProducer getNormsReader() {
return normsReader.get();
}
@Override
public CacheHelper getCoreCacheHelper() {
// same content, we can delegate
return in.getCoreCacheHelper();
}
@Override
public CacheHelper getReaderCacheHelper() {
// same content, we can delegate
return in.getReaderCacheHelper();
}
}

50
lucene/test-framework/src/java/org/apache/lucene/index/MergingDirectoryReaderWrapper.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
import java.io.IOException;
/**
* {@link DirectoryReader} wrapper that uses the merge instances of the wrapped
* {@link CodecReader}s.
* NOTE: This class will fail to work if the leaves of the wrapped directory are
* not codec readers.
*/
public final class MergingDirectoryReaderWrapper extends FilterDirectoryReader {
/** Wrap the given directory. */
public MergingDirectoryReaderWrapper(DirectoryReader in) throws IOException {
super(in, new SubReaderWrapper() {
@Override
public LeafReader wrap(LeafReader reader) {
return new MergingCodecReader((CodecReader) reader);
}
});
}
@Override
protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
return new MergingDirectoryReaderWrapper(in);
}
@Override
public CacheHelper getReaderCacheHelper() {
// doesn't change the content: can delegate
return in.getReaderCacheHelper();
}
}

21
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -1669,7 +1669,7 @@ public abstract class LuceneTestCase extends Assert {
Random random = random();
for (int i = 0, c = random.nextInt(6)+1; i < c; i++) {
switch(random.nextInt(4)) {
switch(random.nextInt(5)) {
case 0:
// will create no FC insanity in atomic case, as ParallelLeafReader has own cache key:
if (VERBOSE) {
@ -1722,6 +1722,25 @@ public abstract class LuceneTestCase extends Assert {
r = new MismatchedDirectoryReader((DirectoryReader)r, random);
}
break;
case 4:
if (VERBOSE) {
System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader=" + r + " with MergingCodecReader");
}
if (r instanceof CodecReader) {
r = new MergingCodecReader((CodecReader) r);
} else if (r instanceof DirectoryReader) {
boolean allLeavesAreCodecReaders = true;
for (LeafReaderContext ctx : r.leaves()) {
if (ctx.reader() instanceof CodecReader == false) {
allLeavesAreCodecReaders = false;
break;
}
}
if (allLeavesAreCodecReaders) {
r = new MergingDirectoryReaderWrapper((DirectoryReader) r);
}
}
break;
default:
fail("should not get here");
}