Improve backwards compatibility tests for sorted indexes. (#2276)

This commit also cleans up some old checks that only applied to pre-6.0 indices.
This commit is contained in:
Julie Tibshirani 2021-02-03 09:27:40 -08:00 committed by GitHub
parent 8a0c1f5a0e
commit 902ce0809d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 186 additions and 172 deletions

View File

@ -60,7 +60,6 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexFormatTooOldException;
@ -93,11 +92,16 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.NormsFieldExistsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
@ -1057,21 +1061,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
TestUtil.checkIndex(dir);
// true if this is a 4.0+ index
final boolean is40Index = FieldInfos.getMergedFieldInfos(reader).fieldInfo("content5") != null;
// true if this is a 4.2+ index
final boolean is42Index =
FieldInfos.getMergedFieldInfos(reader).fieldInfo("dvSortedSet") != null;
// true if this is a 4.9+ index
final boolean is49Index =
FieldInfos.getMergedFieldInfos(reader).fieldInfo("dvSortedNumeric") != null;
// true if this index has points (>= 6.0)
final boolean hasPoints =
FieldInfos.getMergedFieldInfos(reader).fieldInfo("intPoint1d") != null;
assert is40Index;
final Bits liveDocs = MultiBits.getLiveDocs(reader);
assertNotNull(liveDocs);
for (int i = 0; i < 35; i++) {
if (liveDocs.get(i)) {
@ -1079,8 +1070,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
List<IndexableField> fields = d.getFields();
boolean isProxDoc = d.getField("content3") == null;
if (isProxDoc) {
final int numFields = is40Index ? 7 : 5;
assertEquals(numFields, fields.size());
assertEquals(7, fields.size());
IndexableField f = d.getField("id");
assertEquals("" + i, f.stringValue());
@ -1109,87 +1099,78 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
if (is40Index) {
// check docvalues fields
NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");
BinaryDocValues dvBytesDerefFixed =
MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");
BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");
SortedDocValues dvBytesSortedFixed =
MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");
SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");
BinaryDocValues dvBytesStraightFixed =
MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");
BinaryDocValues dvBytesStraightVar =
MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");
NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");
NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");
NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");
NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");
NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");
NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");
SortedSetDocValues dvSortedSet = null;
if (is42Index) {
dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");
}
SortedNumericDocValues dvSortedNumeric = null;
if (is49Index) {
dvSortedNumeric = MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");
}
// check docvalues fields
NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");
BinaryDocValues dvBytesDerefFixed = MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");
BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");
SortedDocValues dvBytesSortedFixed =
MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");
SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");
BinaryDocValues dvBytesStraightFixed =
MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");
BinaryDocValues dvBytesStraightVar =
MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");
NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");
NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");
NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");
NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");
NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");
NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");
for (int i = 0; i < 35; i++) {
int id = Integer.parseInt(reader.document(i).get("id"));
assertEquals(i, dvByte.nextDoc());
assertEquals(id, dvByte.longValue());
SortedSetDocValues dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");
SortedNumericDocValues dvSortedNumeric =
MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");
byte bytes[] =
new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};
BytesRef expectedRef = new BytesRef(bytes);
for (int i = 0; i < 35; i++) {
int id = Integer.parseInt(reader.document(i).get("id"));
assertEquals(i, dvByte.nextDoc());
assertEquals(id, dvByte.longValue());
assertEquals(i, dvBytesDerefFixed.nextDoc());
BytesRef term = dvBytesDerefFixed.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesDerefVar.nextDoc());
term = dvBytesDerefVar.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesSortedFixed.nextDoc());
term = dvBytesSortedFixed.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesSortedVar.nextDoc());
term = dvBytesSortedVar.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesStraightFixed.nextDoc());
term = dvBytesStraightFixed.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesStraightVar.nextDoc());
term = dvBytesStraightVar.binaryValue();
assertEquals(expectedRef, term);
byte bytes[] =
new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};
BytesRef expectedRef = new BytesRef(bytes);
assertEquals(i, dvDouble.nextDoc());
assertEquals((double) id, Double.longBitsToDouble(dvDouble.longValue()), 0D);
assertEquals(i, dvFloat.nextDoc());
assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);
assertEquals(i, dvInt.nextDoc());
assertEquals(id, dvInt.longValue());
assertEquals(i, dvLong.nextDoc());
assertEquals(id, dvLong.longValue());
assertEquals(i, dvPacked.nextDoc());
assertEquals(id, dvPacked.longValue());
assertEquals(i, dvShort.nextDoc());
assertEquals(id, dvShort.longValue());
if (is42Index) {
assertEquals(i, dvSortedSet.nextDoc());
long ord = dvSortedSet.nextOrd();
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
term = dvSortedSet.lookupOrd(ord);
assertEquals(expectedRef, term);
}
if (is49Index) {
assertEquals(i, dvSortedNumeric.nextDoc());
assertEquals(1, dvSortedNumeric.docValueCount());
assertEquals(id, dvSortedNumeric.nextValue());
}
}
assertEquals(i, dvBytesDerefFixed.nextDoc());
BytesRef term = dvBytesDerefFixed.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesDerefVar.nextDoc());
term = dvBytesDerefVar.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesSortedFixed.nextDoc());
term = dvBytesSortedFixed.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesSortedVar.nextDoc());
term = dvBytesSortedVar.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesStraightFixed.nextDoc());
term = dvBytesStraightFixed.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvBytesStraightVar.nextDoc());
term = dvBytesStraightVar.binaryValue();
assertEquals(expectedRef, term);
assertEquals(i, dvDouble.nextDoc());
assertEquals((double) id, Double.longBitsToDouble(dvDouble.longValue()), 0D);
assertEquals(i, dvFloat.nextDoc());
assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);
assertEquals(i, dvInt.nextDoc());
assertEquals(id, dvInt.longValue());
assertEquals(i, dvLong.nextDoc());
assertEquals(id, dvLong.longValue());
assertEquals(i, dvPacked.nextDoc());
assertEquals(id, dvPacked.longValue());
assertEquals(i, dvShort.nextDoc());
assertEquals(id, dvShort.longValue());
assertEquals(i, dvSortedSet.nextDoc());
long ord = dvSortedSet.nextOrd();
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
term = dvSortedSet.lookupOrd(ord);
assertEquals(expectedRef, term);
assertEquals(i, dvSortedNumeric.nextDoc());
assertEquals(1, dvSortedNumeric.docValueCount());
assertEquals(id, dvSortedNumeric.nextValue());
}
ScoreDoc[] hits =
@ -1201,17 +1182,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
doTestHits(hits, 34, searcher.getIndexReader());
if (is40Index) {
hits =
searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), 1000).scoreDocs;
hits = searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), 1000).scoreDocs;
doTestHits(hits, 34, searcher.getIndexReader());
doTestHits(hits, 34, searcher.getIndexReader());
hits =
searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), 1000).scoreDocs;
hits = searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), 1000).scoreDocs;
doTestHits(hits, 34, searcher.getIndexReader());
}
doTestHits(hits, 34, searcher.getIndexReader());
hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), 1000).scoreDocs;
assertEquals(34, hits.length);
@ -1223,72 +1200,67 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), 1000).scoreDocs;
assertEquals(34, hits.length);
if (hasPoints) {
doTestHits(
searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)
.scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
FloatPoint.newRangeQuery(
"floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
DoublePoint.newRangeQuery(
"doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)
.scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
FloatPoint.newRangeQuery(
"floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,
34,
searcher.getIndexReader());
doTestHits(
searcher.search(
DoublePoint.newRangeQuery(
"doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
byte[] bytes1 = new byte[4];
byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};
doTestHits(
searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000)
.scoreDocs,
34,
searcher.getIndexReader());
byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};
doTestHits(
searcher.search(
BinaryPoint.newRangeQuery(
"binaryPoint2d",
new byte[][] {bytes1, bytes1},
new byte[][] {bytes2, bytes3}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
}
byte[] bytes1 = new byte[4];
byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};
doTestHits(
searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000).scoreDocs,
34,
searcher.getIndexReader());
byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};
doTestHits(
searcher.search(
BinaryPoint.newRangeQuery(
"binaryPoint2d", new byte[][] {bytes1, bytes1}, new byte[][] {bytes2, bytes3}),
1000)
.scoreDocs,
34,
searcher.getIndexReader());
reader.close();
}
@ -1791,9 +1763,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
Path oldIndexDir = createTempDir("moreterms");
TestUtil.unzip(getDataInputStream(moreTermsIndex), oldIndexDir);
Directory dir = newFSDirectory(oldIndexDir);
DirectoryReader reader = DirectoryReader.open(dir);
verifyUsesDefaultCodec(dir, moreTermsIndex);
// TODO: more tests
TestUtil.checkIndex(dir);
searchExampleIndex(reader);
reader.close();
dir.close();
}
@ -1978,22 +1954,60 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
assertNotNull("Sorted index index " + name + " not found", resource);
TestUtil.unzip(resource, path);
// TODO: more tests
Directory dir = newFSDirectory(path);
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
assertNotNull(sort);
assertEquals("<long: \"dateDV\">!", sort.toString());
reader.close();
// this will confirm the docs really are sorted:
// This will confirm the docs are really sorted
TestUtil.checkIndex(dir);
searchExampleIndex(reader);
reader.close();
dir.close();
}
}
private void searchExampleIndex(DirectoryReader reader) throws IOException {
IndexSearcher searcher = newSearcher(reader);
TopDocs topDocs = searcher.search(new NormsFieldExistsQuery("titleTokenized"), 10);
assertEquals(50, topDocs.totalHits.value);
topDocs = searcher.search(new DocValuesFieldExistsQuery("titleDV"), 10);
assertEquals(50, topDocs.totalHits.value);
topDocs = searcher.search(new TermQuery(new Term("body", "ja")), 10);
assertTrue(topDocs.totalHits.value > 0);
topDocs =
searcher.search(
IntPoint.newRangeQuery("docid_int", 42, 44),
10,
new Sort(new SortField("docid_intDV", SortField.Type.INT)));
assertEquals(3, topDocs.totalHits.value);
assertEquals(3, topDocs.scoreDocs.length);
assertEquals(42, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);
assertEquals(43, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);
assertEquals(44, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);
topDocs = searcher.search(new TermQuery(new Term("body", "the")), 5);
assertTrue(topDocs.totalHits.value > 0);
topDocs =
searcher.search(
new MatchAllDocsQuery(), 5, new Sort(new SortField("dateDV", SortField.Type.LONG)));
assertEquals(50, topDocs.totalHits.value);
assertEquals(5, topDocs.scoreDocs.length);
long firstDate = (Long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0];
long lastDate = (Long) ((FieldDoc) topDocs.scoreDocs[4]).fields[0];
assertTrue(firstDate <= lastDate);
}
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
int idx = term.offset;