Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

beef up tests 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5178@1515686 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2013-08-20 02:05:12 +00:00
parent 3b49cf1c86
commit cf0078a686
6 changed files with 352 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
View File

@ -28,7 +28,6 @@ import java.util.Locale;
import java.util.Map;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.DocValuesProducer.SortedSetDocsWithField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
@ -431,16 +430,17 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
@Override
public Bits getDocsWithField(FieldInfo field) throws IOException {
if (field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_SET) {
return new SortedSetDocsWithField(getSortedSet(field), maxDoc);
} else if (field.getDocValuesType() == FieldInfo.DocValuesType.SORTED) {
return new SortedDocsWithField(getSorted(field), maxDoc);
} else if (field.getDocValuesType() == FieldInfo.DocValuesType.BINARY) {
return getBinaryDocsWithField(field);
} else if (field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC) {
return getNumericDocsWithField(field);
} else {
return new Bits.MatchAllBits(maxDoc);
switch (field.getDocValuesType()) {
case SORTED_SET:
return new SortedSetDocsWithField(getSortedSet(field), maxDoc);
case SORTED:
return new SortedDocsWithField(getSorted(field), maxDoc);
case BINARY:
return getBinaryDocsWithField(field);
case NUMERIC:
return getNumericDocsWithField(field);
default:
throw new AssertionError();
}
}

88
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
View File

@ -175,28 +175,6 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
checkIterator(ords.iterator(), ordCount, false);
in.addSortedSetField(field, values, docToOrdCount, ords);
}
private <T> void checkIterator(Iterator<T> iterator, long expectedSize, boolean allowNull) {
for (long i = 0; i < expectedSize; i++) {
boolean hasNext = iterator.hasNext();
assert hasNext;
T v = iterator.next();
assert allowNull || v != null;
try {
iterator.remove();
throw new AssertionError("broken iterator (supports remove): " + iterator);
} catch (UnsupportedOperationException expected) {
// ok
}
}
assert !iterator.hasNext();
try {
iterator.next();
throw new AssertionError("broken iterator (allows next() when hasNext==false) " + iterator);
} catch (NoSuchElementException expected) {
// ok
}
}
@Override
public void close() throws IOException {
@ -204,6 +182,70 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
}
}
static class AssertingNormsConsumer extends DocValuesConsumer {
private final DocValuesConsumer in;
private final int maxDoc;
AssertingNormsConsumer(DocValuesConsumer in, int maxDoc) {
this.in = in;
this.maxDoc = maxDoc;
}
@Override
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
int count = 0;
for (Number v : values) {
assert v != null;
count++;
}
assert count == maxDoc;
checkIterator(values.iterator(), maxDoc, false);
in.addNumericField(field, values);
}
@Override
public void close() throws IOException {
in.close();
}
@Override
public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
throw new IllegalStateException();
}
@Override
public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
throw new IllegalStateException();
}
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
throw new IllegalStateException();
}
}
private static <T> void checkIterator(Iterator<T> iterator, long expectedSize, boolean allowNull) {
for (long i = 0; i < expectedSize; i++) {
boolean hasNext = iterator.hasNext();
assert hasNext;
T v = iterator.next();
assert allowNull || v != null;
try {
iterator.remove();
throw new AssertionError("broken iterator (supports remove): " + iterator);
} catch (UnsupportedOperationException expected) {
// ok
}
}
assert !iterator.hasNext();
try {
iterator.next();
throw new AssertionError("broken iterator (allows next() when hasNext==false) " + iterator);
} catch (NoSuchElementException expected) {
// ok
}
}
static class AssertingDocValuesProducer extends DocValuesProducer {
private final DocValuesProducer in;
private final int maxDoc;
@ -252,7 +294,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
Bits bits = in.getDocsWithField(field);
assert bits != null;
assert bits.length() == maxDoc;
return bits; // TODO: add AssertingBits w/ bounds check
return new AssertingAtomicReader.AssertingBits(bits);
}
@Override

4
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
View File

@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingDocValuesConsumer;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingNormsConsumer;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingDocValuesProducer;
import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.index.SegmentReadState;
@ -38,7 +38,7 @@ public class AssertingNormsFormat extends NormsFormat {
public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
DocValuesConsumer consumer = in.normsConsumer(state);
assert consumer != null;
return new AssertingDocValuesConsumer(consumer, state.segmentInfo.getDocCount());
return new AssertingNormsConsumer(consumer, state.segmentInfo.getDocCount());
}
@Override

22
lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
View File

@ -607,12 +607,33 @@ public class AssertingAtomicReader extends FilterAtomicReader {
return null;
}
}
/** Wraps a Bits but with additional asserts */
public static class AssertingBits implements Bits {
final Bits in;
public AssertingBits(Bits in) {
this.in = in;
}
@Override
public boolean get(int index) {
assert index >= 0 && index < length();
return in.get(index);
}
@Override
public int length() {
return in.length();
}
}
@Override
public Bits getLiveDocs() {
Bits liveDocs = super.getLiveDocs();
if (liveDocs != null) {
assert maxDoc() == liveDocs.length();
liveDocs = new AssertingBits(liveDocs);
} else {
assert maxDoc() == numDocs();
assert !hasDeletions();
@ -628,6 +649,7 @@ public class AssertingAtomicReader extends FilterAtomicReader {
assert fi != null;
assert fi.hasDocValues();
assert maxDoc() == docsWithField.length();
docsWithField = new AssertingBits(docsWithField);
} else {
assert fi == null || fi.hasDocValues() == false;
}

258
lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
View File

@ -26,6 +26,7 @@ import java.util.Map.Entry;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@ -673,7 +674,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
iwriter.close();
SortedDocValues dv = getOnlySegmentReader(ireader).getSortedDocValues("field");
if (codecSupportsDocsWithField("field")) {
if (defaultCodecSupportsDocsWithField()) {
assertEquals(-1, dv.getOrd(0));
assertEquals(0, dv.getValueCount());
} else {
@ -733,7 +734,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
BytesRef scratch = new BytesRef();
dv.lookupOrd(dv.getOrd(0), scratch);
assertEquals(new BytesRef("hello world 2"), scratch);
if (codecSupportsDocsWithField("dv")) {
if (defaultCodecSupportsDocsWithField()) {
assertEquals(-1, dv.getOrd(1));
}
dv.get(1, scratch);
@ -1115,7 +1116,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
doc.add(newTextField("id", "noValue", Field.Store.YES));
w.addDocument(doc);
}
if (!codecSupportsDocsWithField("field")) {
if (!defaultCodecSupportsDocsWithField()) {
BytesRef bytesRef = new BytesRef();
hash.add(bytesRef); // add empty value for the gaps
}
@ -1242,7 +1243,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
}
private void doTestMissingVsFieldCache(LongProducer longs) throws Exception {
assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField("dv"));
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
@ -2345,7 +2346,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
}
public void testTwoNumbersOneMissing() throws IOException {
assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField("dv1"));
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setMergePolicy(newLogMergePolicy());
@ -2374,7 +2375,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
}
public void testTwoNumbersOneMissingWithMerging() throws IOException {
assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField("dv1"));
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setMergePolicy(newLogMergePolicy());
@ -2404,7 +2405,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
}
public void testThreeNumbersOneMissingWithMerging() throws IOException {
assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField("dv1"));
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setMergePolicy(newLogMergePolicy());
@ -2440,7 +2441,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
}
public void testTwoBytesOneMissing() throws IOException {
assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField("dv1"));
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setMergePolicy(newLogMergePolicy());
@ -2472,7 +2473,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
}
public void testTwoBytesOneMissingWithMerging() throws IOException {
assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField("dv1"));
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setMergePolicy(newLogMergePolicy());
@ -2505,7 +2506,7 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
}
public void testThreeBytesOneMissingWithMerging() throws IOException {
assumeTrue("Codec does not support getDocsWithField", codecSupportsDocsWithField("dv1"));
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setMergePolicy(newLogMergePolicy());
@ -2709,6 +2710,243 @@ public abstract class BaseDocValuesFormatTestCase extends LuceneTestCase {
ar.close();
d.close();
}
/** Tests dv against stored fields with threads (binary/numeric/sorted, no missing) */
public void testThreads() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
Document doc = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
Field storedBinField = new StoredField("storedBin", new byte[0]);
Field dvBinField = new BinaryDocValuesField("dvBin", new BytesRef());
Field dvSortedField = new SortedDocValuesField("dvSorted", new BytesRef());
Field storedNumericField = new StoredField("storedNum", "");
Field dvNumericField = new NumericDocValuesField("dvNum", 0);
doc.add(idField);
doc.add(storedBinField);
doc.add(dvBinField);
doc.add(dvSortedField);
doc.add(storedNumericField);
doc.add(dvNumericField);
// index some docs
int numDocs = atLeast(300);
for (int i = 0; i < numDocs; i++) {
idField.setStringValue(Integer.toString(i));
int length = _TestUtil.nextInt(random(), 0, 8);
byte buffer[] = new byte[length];
random().nextBytes(buffer);
storedBinField.setBytesValue(buffer);
dvBinField.setBytesValue(buffer);
dvSortedField.setBytesValue(buffer);
long numericValue = random().nextLong();
storedNumericField.setStringValue(Long.toString(numericValue));
dvNumericField.setLongValue(numericValue);
writer.addDocument(doc);
if (random().nextInt(31) == 0) {
writer.commit();
}
}
// delete some docs
int numDeletions = random().nextInt(numDocs/10);
for (int i = 0; i < numDeletions; i++) {
int id = random().nextInt(numDocs);
writer.deleteDocuments(new Term("id", Integer.toString(id)));
}
writer.close();
// compare
final DirectoryReader ir = DirectoryReader.open(dir);
int numThreads = _TestUtil.nextInt(random(), 2, 7);
Thread threads[] = new Thread[numThreads];
final CountDownLatch startingGun = new CountDownLatch(1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread() {
@Override
public void run() {
try {
startingGun.await();
for (AtomicReaderContext context : ir.leaves()) {
AtomicReader r = context.reader();
BinaryDocValues binaries = r.getBinaryDocValues("dvBin");
SortedDocValues sorted = r.getSortedDocValues("dvSorted");
NumericDocValues numerics = r.getNumericDocValues("dvNum");
for (int j = 0; j < r.maxDoc(); j++) {
BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
BytesRef scratch = new BytesRef();
binaries.get(j, scratch);
assertEquals(binaryValue, scratch);
sorted.get(j, scratch);
assertEquals(binaryValue, scratch);
String expected = r.document(j).get("storedNum");
assertEquals(Long.parseLong(expected), numerics.get(j));
}
}
_TestUtil.checkReader(ir);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
threads[i].start();
}
startingGun.countDown();
for (Thread t : threads) {
t.join();
}
ir.close();
dir.close();
}
/** Tests dv against stored fields with threads (all types + missing) */
public void testThreads2() throws Exception {
assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
Field idField = new StringField("id", "", Field.Store.NO);
Field storedBinField = new StoredField("storedBin", new byte[0]);
Field dvBinField = new BinaryDocValuesField("dvBin", new BytesRef());
Field dvSortedField = new SortedDocValuesField("dvSorted", new BytesRef());
Field storedNumericField = new StoredField("storedNum", "");
Field dvNumericField = new NumericDocValuesField("dvNum", 0);
// index some docs
int numDocs = atLeast(300);
for (int i = 0; i < numDocs; i++) {
idField.setStringValue(Integer.toString(i));
int length = _TestUtil.nextInt(random(), 0, 8);
byte buffer[] = new byte[length];
random().nextBytes(buffer);
storedBinField.setBytesValue(buffer);
dvBinField.setBytesValue(buffer);
dvSortedField.setBytesValue(buffer);
long numericValue = random().nextLong();
storedNumericField.setStringValue(Long.toString(numericValue));
dvNumericField.setLongValue(numericValue);
Document doc = new Document();
doc.add(idField);
if (random().nextInt(4) > 0) {
doc.add(storedBinField);
doc.add(dvBinField);
doc.add(dvSortedField);
}
if (random().nextInt(4) > 0) {
doc.add(storedNumericField);
doc.add(dvNumericField);
}
int numSortedSetFields = random().nextInt(3);
Set<String> values = new TreeSet<String>();
for (int j = 0; j < numSortedSetFields; j++) {
values.add(_TestUtil.randomSimpleString(random()));
}
for (String v : values) {
doc.add(new SortedSetDocValuesField("dvSortedSet", new BytesRef(v)));
doc.add(new StoredField("storedSortedSet", v));
}
writer.addDocument(doc);
if (random().nextInt(31) == 0) {
writer.commit();
}
}
// delete some docs
int numDeletions = random().nextInt(numDocs/10);
for (int i = 0; i < numDeletions; i++) {
int id = random().nextInt(numDocs);
writer.deleteDocuments(new Term("id", Integer.toString(id)));
}
writer.close();
// compare
final DirectoryReader ir = DirectoryReader.open(dir);
int numThreads = _TestUtil.nextInt(random(), 2, 7);
Thread threads[] = new Thread[numThreads];
final CountDownLatch startingGun = new CountDownLatch(1);
for (int i = 0; i < threads.length; i++) {
threads[i] = new Thread() {
@Override
public void run() {
try {
startingGun.await();
for (AtomicReaderContext context : ir.leaves()) {
AtomicReader r = context.reader();
BinaryDocValues binaries = r.getBinaryDocValues("dvBin");
Bits binaryBits = r.getDocsWithField("dvBin");
SortedDocValues sorted = r.getSortedDocValues("dvSorted");
Bits sortedBits = r.getDocsWithField("dvSorted");
NumericDocValues numerics = r.getNumericDocValues("dvNum");
Bits numericBits = r.getDocsWithField("dvNum");
SortedSetDocValues sortedSet = r.getSortedSetDocValues("dvSortedSet");
Bits sortedSetBits = r.getDocsWithField("dvSortedSet");
for (int j = 0; j < r.maxDoc(); j++) {
BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
if (binaryValue != null) {
if (binaries != null) {
BytesRef scratch = new BytesRef();
binaries.get(j, scratch);
assertEquals(binaryValue, scratch);
sorted.get(j, scratch);
assertEquals(binaryValue, scratch);
assertTrue(binaryBits.get(j));
assertTrue(sortedBits.get(j));
}
} else if (binaries != null) {
assertFalse(binaryBits.get(j));
assertFalse(sortedBits.get(j));
assertEquals(-1, sorted.getOrd(j));
}
String number = r.document(j).get("storedNum");
if (number != null) {
if (numerics != null) {
assertEquals(Long.parseLong(number), numerics.get(j));
}
} else if (numerics != null) {
assertFalse(numericBits.get(j));
assertEquals(0, numerics.get(j));
}
String values[] = r.document(j).getValues("storedSortedSet");
if (values.length > 0) {
assertNotNull(sortedSet);
sortedSet.setDocument(j);
for (int i = 0; i < values.length; i++) {
long ord = sortedSet.nextOrd();
assertTrue(ord != SortedSetDocValues.NO_MORE_ORDS);
BytesRef value = new BytesRef();
sortedSet.lookupOrd(ord, value);
assertEquals(values[i], value.utf8ToString());
}
assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
assertTrue(sortedSetBits.get(j));
} else if (sortedSet != null) {
sortedSet.setDocument(j);
assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
assertFalse(sortedSetBits.get(j));
}
}
}
_TestUtil.checkReader(ir);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
threads[i].start();
}
startingGun.countDown();
for (Thread t : threads) {
t.join();
}
ir.close();
dir.close();
}
protected boolean codecAcceptsHugeBinaryValues(String field) {
return true;

8
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -1369,11 +1369,11 @@ public abstract class LuceneTestCase extends Assert {
return true;
}
/** Returns true if the codec for the field "supports" docsWithField
/** Returns true if the codec "supports" docsWithField
* (other codecs return MatchAllBits, because you couldnt write missing values before) */
public static boolean codecSupportsDocsWithField(String field) {
String name = _TestUtil.getDocValuesFormat(Codec.getDefault(), field);
if (name.equals("Lucene40") || name.equals("Lucene42")) {
public static boolean defaultCodecSupportsDocsWithField() {
String name = Codec.getDefault().getName();
if (name.equals("Lucene40") || name.equals("Lucene41") || name.equals("Lucene42")) {
return false;
}
return true;