LUCENE-5666: fix some tests

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5666@1594311 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-05-13 18:08:30 +00:00
parent c58788c27a
commit 37d3efbc62
6 changed files with 301 additions and 37 deletions

View File

@ -0,0 +1,248 @@
package org.apache.lucene.uninverting;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NumericUtils;
public class TestUninvertingReader extends LuceneTestCase {
public void testSortedSetInteger() throws IOException {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
Document doc = new Document();
doc.add(new IntField("foo", 5, Field.Store.NO));
iw.addDocument(doc);
doc = new Document();
doc.add(new IntField("foo", 5, Field.Store.NO));
doc.add(new IntField("foo", -3, Field.Store.NO));
iw.addDocument(doc);
iw.forceMerge(1);
iw.shutdown();
DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
Collections.singletonMap("foo", Type.SORTED_SET_INTEGER));
AtomicReader ar = ir.leaves().get(0).reader();
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
assertEquals(2, v.getValueCount());
v.setDocument(0);
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
v.setDocument(1);
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
assertEquals(-3, NumericUtils.prefixCodedToInt(value));
v.lookupOrd(1, value);
assertEquals(5, NumericUtils.prefixCodedToInt(value));
ir.close();
dir.close();
}
public void testSortedSetFloat() throws IOException {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
Document doc = new Document();
doc.add(new IntField("foo", Float.floatToRawIntBits(5f), Field.Store.NO));
iw.addDocument(doc);
doc = new Document();
doc.add(new IntField("foo", Float.floatToRawIntBits(5f), Field.Store.NO));
doc.add(new IntField("foo", Float.floatToRawIntBits(-3f), Field.Store.NO));
iw.addDocument(doc);
iw.forceMerge(1);
iw.shutdown();
DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
Collections.singletonMap("foo", Type.SORTED_SET_FLOAT));
AtomicReader ar = ir.leaves().get(0).reader();
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
assertEquals(2, v.getValueCount());
v.setDocument(0);
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
v.setDocument(1);
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
assertEquals(Float.floatToRawIntBits(-3f), NumericUtils.prefixCodedToInt(value));
v.lookupOrd(1, value);
assertEquals(Float.floatToRawIntBits(5f), NumericUtils.prefixCodedToInt(value));
ir.close();
dir.close();
}
public void testSortedSetLong() throws IOException {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
Document doc = new Document();
doc.add(new LongField("foo", 5, Field.Store.NO));
iw.addDocument(doc);
doc = new Document();
doc.add(new LongField("foo", 5, Field.Store.NO));
doc.add(new LongField("foo", -3, Field.Store.NO));
iw.addDocument(doc);
iw.forceMerge(1);
iw.shutdown();
DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
Collections.singletonMap("foo", Type.SORTED_SET_LONG));
AtomicReader ar = ir.leaves().get(0).reader();
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
assertEquals(2, v.getValueCount());
v.setDocument(0);
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
v.setDocument(1);
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
assertEquals(-3, NumericUtils.prefixCodedToLong(value));
v.lookupOrd(1, value);
assertEquals(5, NumericUtils.prefixCodedToLong(value));
ir.close();
dir.close();
}
public void testSortedSetDouble() throws IOException {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
Document doc = new Document();
doc.add(new LongField("foo", Double.doubleToRawLongBits(5d), Field.Store.NO));
iw.addDocument(doc);
doc = new Document();
doc.add(new LongField("foo", Double.doubleToRawLongBits(5d), Field.Store.NO));
doc.add(new LongField("foo", Double.doubleToRawLongBits(-3d), Field.Store.NO));
iw.addDocument(doc);
iw.forceMerge(1);
iw.shutdown();
DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
Collections.singletonMap("foo", Type.SORTED_SET_DOUBLE));
AtomicReader ar = ir.leaves().get(0).reader();
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
SortedSetDocValues v = ar.getSortedSetDocValues("foo");
assertNoSilentInsanity(ar, "foo", DocValuesType.SORTED_SET);
assertEquals(2, v.getValueCount());
v.setDocument(0);
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
v.setDocument(1);
assertEquals(0, v.nextOrd());
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
assertEquals(Double.doubleToRawLongBits(-3d), NumericUtils.prefixCodedToLong(value));
v.lookupOrd(1, value);
assertEquals(Double.doubleToRawLongBits(5d), NumericUtils.prefixCodedToLong(value));
ir.close();
dir.close();
}
private void assertNoSilentInsanity(AtomicReader reader, String field, DocValuesType type) throws IOException {
Set<DocValuesType> insaneTypes = EnumSet.allOf(DocValuesType.class);
insaneTypes.remove(type);
for (DocValuesType t : insaneTypes) {
tryToBeInsane(reader, field, type, t);
}
}
private void tryToBeInsane(AtomicReader reader, String field, DocValuesType actualType, DocValuesType insaneType) throws IOException {
try {
switch(insaneType) {
case NUMERIC:
reader.getNumericDocValues(field);
break;
case SORTED:
reader.getSortedDocValues(field);
break;
case BINARY:
reader.getBinaryDocValues(field);
break;
case SORTED_SET:
reader.getSortedSetDocValues(field);
default:
throw new AssertionError();
}
fail("didn't get expected exception: actual=" + actualType + ",insane=" + insaneType);
} catch (IllegalStateException expected) {}
}
}

View File

@ -120,6 +120,7 @@ public class PreAnalyzedField extends FieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
return new SortedSetSortField(field.getName(), top);
}

View File

@ -96,6 +96,7 @@ public class TextField extends FieldType {
@Override
public SortField getSortField(SchemaField field, boolean reverse) {
/* :TODO: maybe warn if isTokenized(), but doesn't use LimitTokenCountFilter in it's chain? */
field.checkSortability();
return new SortedSetSortField(field.getName(), reverse);
}

View File

@ -369,7 +369,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
long cpg = reader.getIndexCommit().getGeneration();
try {
if (closeReader) reader.decRef();
if (closeReader) rawReader.decRef();
} catch (Exception e) {
SolrException.log(log, "Problem dec ref'ing reader", e);
}

View File

@ -22,6 +22,9 @@ import java.util.List;
import java.util.Locale;
import java.util.Random;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.uninverting.DocTermOrds;
@ -81,12 +84,11 @@ public class TestFaceting extends SolrTestCaseJ4 {
createIndex(size);
req = lrf.makeRequest("q","*:*");
UnInvertedField uif = new UnInvertedField(proto.field(), req.getSearcher());
SortedSetDocValues dv = DocValues.getSortedSet(req.getSearcher().getAtomicReader(), proto.field());
assertEquals(size, uif.getNumTerms());
assertEquals(size, dv.getValueCount());
TermsEnum te = uif.getOrdTermsEnum(req.getSearcher().getAtomicReader());
assertEquals(size == 0, te == null);
TermsEnum te = dv.termsEnum();
Random r = new Random(size);
// test seeking by term string
@ -763,16 +765,16 @@ public class TestFaceting extends SolrTestCaseJ4 {
RefCounted<SolrIndexSearcher> currentSearcherRef = h.getCore().getSearcher();
try {
SolrIndexSearcher currentSearcher = currentSearcherRef.get();
UnInvertedField ui0 = UnInvertedField.getUnInvertedField("f0_ws", currentSearcher);
UnInvertedField ui1 = UnInvertedField.getUnInvertedField("f1_ws", currentSearcher);
UnInvertedField ui2 = UnInvertedField.getUnInvertedField("f2_ws", currentSearcher);
UnInvertedField ui3 = UnInvertedField.getUnInvertedField("f3_ws", currentSearcher);
UnInvertedField ui4 = UnInvertedField.getUnInvertedField("f4_ws", currentSearcher);
UnInvertedField ui5 = UnInvertedField.getUnInvertedField("f5_ws", currentSearcher);
UnInvertedField ui6 = UnInvertedField.getUnInvertedField("f6_ws", currentSearcher);
UnInvertedField ui7 = UnInvertedField.getUnInvertedField("f7_ws", currentSearcher);
UnInvertedField ui8 = UnInvertedField.getUnInvertedField("f8_ws", currentSearcher);
UnInvertedField ui9 = UnInvertedField.getUnInvertedField("f9_ws", currentSearcher);
SortedSetDocValues ui0 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f0_ws");
SortedSetDocValues ui1 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f1_ws");
SortedSetDocValues ui2 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f2_ws");
SortedSetDocValues ui3 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f3_ws");
SortedSetDocValues ui4 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f4_ws");
SortedSetDocValues ui5 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f5_ws");
SortedSetDocValues ui6 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f6_ws");
SortedSetDocValues ui7 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f7_ws");
SortedSetDocValues ui8 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f8_ws");
SortedSetDocValues ui9 = DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f9_ws");
assertQ("check threading, more threads than fields",
req("q", "id:*", "indent", "true", "fl", "id", "rows", "1"
@ -924,28 +926,39 @@ public class TestFaceting extends SolrTestCaseJ4 {
// Now, are all the UnInvertedFields still the same? Meaning they weren't re-fetched even when a bunch were
// requested at the same time?
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui0, UnInvertedField.getUnInvertedField("f0_ws", currentSearcher));
ui0, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f0_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui1, UnInvertedField.getUnInvertedField("f1_ws", currentSearcher));
ui1, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f1_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui2, UnInvertedField.getUnInvertedField("f2_ws", currentSearcher));
ui2, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f2_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui3, UnInvertedField.getUnInvertedField("f3_ws", currentSearcher));
ui3, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f3_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui4, UnInvertedField.getUnInvertedField("f4_ws", currentSearcher));
ui4, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f4_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui5, UnInvertedField.getUnInvertedField("f5_ws", currentSearcher));
ui5, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f5_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui6, UnInvertedField.getUnInvertedField("f6_ws", currentSearcher));
ui6, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f6_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui7, UnInvertedField.getUnInvertedField("f7_ws", currentSearcher));
ui7, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f7_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui8, UnInvertedField.getUnInvertedField("f8_ws", currentSearcher));
ui8, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f8_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui9, UnInvertedField.getUnInvertedField("f9_ws", currentSearcher));
ui9, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f9_ws"));
} finally {
currentSearcherRef.decref();
}
}
// assert same instance: either same object, or both wrapping same single-valued object
private void assertEquals(String msg, SortedSetDocValues dv1, SortedSetDocValues dv2) {
SortedDocValues singleton1 = DocValues.unwrapSingleton(dv1);
SortedDocValues singleton2 = DocValues.unwrapSingleton(dv2);
if (singleton1 == null || singleton2 == null) {
assertSame(dv1, dv2);
} else {
assertSame(singleton1, singleton2);
}
}
}

View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReaderContext;
@ -71,7 +72,7 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
assertU(commit());
SolrQueryRequest sr1 = req("q","foo");
IndexReaderContext rCtx1 = sr1.getSearcher().getTopReaderContext();
IndexReader r1 = sr1.getSearcher().getRawReader();
String sval1 = getStringVal(sr1, "v_s1",0);
assertEquals("string1", sval1);
@ -81,28 +82,28 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
assertU(commit());
SolrQueryRequest sr2 = req("q","foo");
IndexReaderContext rCtx2 = sr2.getSearcher().getTopReaderContext();
IndexReader r2 = sr2.getSearcher().getRawReader();
// make sure the readers share the first segment
// Didn't work w/ older versions of lucene2.9 going from segment -> multi
assertEquals(rCtx1.leaves().get(0).reader(), rCtx2.leaves().get(0).reader());
assertEquals(r1.leaves().get(0).reader(), r2.leaves().get(0).reader());
assertU(adoc("id","5", "v_f","3.14159"));
assertU(adoc("id","6", "v_f","8983", "v_s1","string6"));
assertU(commit());
SolrQueryRequest sr3 = req("q","foo");
IndexReaderContext rCtx3 = sr3.getSearcher().getTopReaderContext();
IndexReader r3 = sr3.getSearcher().getRawReader();
// make sure the readers share segments
// assertEquals(r1.getLeafReaders()[0], r3.getLeafReaders()[0]);
assertEquals(rCtx2.leaves().get(0).reader(), rCtx3.leaves().get(0).reader());
assertEquals(rCtx2.leaves().get(1).reader(), rCtx3.leaves().get(1).reader());
assertEquals(r2.leaves().get(0).reader(), r3.leaves().get(0).reader());
assertEquals(r2.leaves().get(1).reader(), r3.leaves().get(1).reader());
sr1.close();
sr2.close();
// should currently be 1, but this could change depending on future index management
int baseRefCount = rCtx3.reader().getRefCount();
int baseRefCount = r3.getRefCount();
assertEquals(1, baseRefCount);
Object sr3SearcherRegAt = sr3.getSearcher().getStatistics().get("registeredAt");
@ -112,7 +113,7 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
sr3.getSearcher(), sr4.getSearcher());
assertEquals("nothing changed, searcher should not have been re-registered",
sr3SearcherRegAt, sr4.getSearcher().getStatistics().get("registeredAt"));
IndexReaderContext rCtx4 = sr4.getSearcher().getTopReaderContext();
IndexReader r4 = sr4.getSearcher().getRawReader();
// force an index change so the registered searcher won't be the one we are testing (and
// then we should be able to test the refCount going all the way to 0
@ -120,12 +121,12 @@ public class TestIndexSearcher extends SolrTestCaseJ4 {
assertU(commit());
// test that reader didn't change
assertSame(rCtx3.reader(), rCtx4.reader());
assertEquals(baseRefCount, rCtx4.reader().getRefCount());
assertSame(r3, r4);
assertEquals(baseRefCount, r4.getRefCount());
sr3.close();
assertEquals(baseRefCount, rCtx4.reader().getRefCount());
assertEquals(baseRefCount, r4.getRefCount());
sr4.close();
assertEquals(baseRefCount-1, rCtx4.reader().getRefCount());
assertEquals(baseRefCount-1, r4.getRefCount());
SolrQueryRequest sr5 = req("q","foo");