Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1461: more careful checking of lower/upper bound to the FieldCacheRangeFilter 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@722203 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2008-12-01 19:52:54 +00:00
parent 278dc0fc04
commit e3d5b53e53
2 changed files with 117 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
View File

@ -123,15 +123,27 @@ public class FieldCacheRangeFilter extends Filter {
private void initialize() { private void initialize() {
int lowerPoint = fcsi.binarySearchLookup(lowerVal); int lowerPoint = fcsi.binarySearchLookup(lowerVal);
if (includeLower && lowerPoint >= 0) { int upperPoint = fcsi.binarySearchLookup(upperVal);
if (lowerPoint == 0 && upperPoint == 0) {
throw new IllegalArgumentException("At least one value must be non-null");
}
if (includeLower && lowerPoint == 0) {
throw new IllegalArgumentException("The lower bound must be non-null to be inclusive");
} else if (includeLower && lowerPoint > 0) {
inclusiveLowerPoint = lowerPoint; inclusiveLowerPoint = lowerPoint;
} else if (lowerPoint >= 0) { } else if (lowerPoint >= 0) {
inclusiveLowerPoint = lowerPoint+1; inclusiveLowerPoint = lowerPoint+1;
} else { } else {
inclusiveLowerPoint = -lowerPoint-1; inclusiveLowerPoint = -lowerPoint-1;
} }
int upperPoint = fcsi.binarySearchLookup(upperVal);
if (includeUpper && upperPoint >= 0) { if (includeUpper && upperPoint == 0) {
throw new IllegalArgumentException("The upper bound must be non-null to be inclusive");
} else if (upperPoint == 0) {
inclusiveUpperPoint = Integer.MAX_VALUE;
} else if (includeUpper && upperPoint > 0) {
inclusiveUpperPoint = upperPoint; inclusiveUpperPoint = upperPoint;
} else if (upperPoint >= 0) { } else if (upperPoint >= 0) {
inclusiveUpperPoint = upperPoint - 1; inclusiveUpperPoint = upperPoint - 1;

391
src/test/org/apache/lucene/search/TestFieldCacheRangeFilter.java
View File

@ -40,340 +40,153 @@ import org.apache.lucene.store.RAMDirectory;
*/ */
public class TestFieldCacheRangeFilter extends BaseTestRangeFilter { public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
public TestFieldCacheRangeFilter(String name) { public TestFieldCacheRangeFilter(String name) {
super(name); super(name);
} }
public TestFieldCacheRangeFilter() { public TestFieldCacheRangeFilter() {
super(); super();
} }
public void testRangeFilterId() throws IOException { public void testRangeFilterId() throws IOException {
IndexReader reader = IndexReader.open(signedIndex.index); IndexReader reader = IndexReader.open(signedIndex.index);
IndexSearcher search = new IndexSearcher(reader); IndexSearcher search = new IndexSearcher(reader);
int medId = ((maxId - minId) / 2); int medId = ((maxId - minId) / 2);
String minIP = pad(minId); String minIP = pad(minId);
String maxIP = pad(maxId); String maxIP = pad(maxId);
String medIP = pad(medId); String medIP = pad(medId);
int numDocs = reader.numDocs(); int numDocs = reader.numDocs();
assertEquals("num of docs", numDocs, 1+ maxId - minId); assertEquals("num of docs", numDocs, 1+ maxId - minId);
ScoreDoc[] result; ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body")); Query q = new TermQuery(new Term("body","body"));
// test id, bounded on both ends // test id, bounded on both ends
result = search.search(q,new RangeFilter("id",minIP,maxIP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,maxIP,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length); assertEquals("find all", numDocs, result.length);
result = search.search(q,new RangeFilter("id",minIP,maxIP,T,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,maxIP,T,F), numDocs).scoreDocs;
assertEquals("all but last", numDocs-1, result.length); assertEquals("all but last", numDocs-1, result.length);
result = search.search(q,new RangeFilter("id",minIP,maxIP,F,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,maxIP,F,T), numDocs).scoreDocs;
assertEquals("all but first", numDocs-1, result.length); assertEquals("all but first", numDocs-1, result.length);
result = search.search(q,new RangeFilter("id",minIP,maxIP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,maxIP,F,F), numDocs).scoreDocs;
assertEquals("all but ends", numDocs-2, result.length); assertEquals("all but ends", numDocs-2, result.length);
result = search.search(q,new RangeFilter("id",medIP,maxIP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",medIP,maxIP,T,T), numDocs).scoreDocs;
assertEquals("med and up", 1+ maxId-medId, result.length); assertEquals("med and up", 1+ maxId-medId, result.length);
result = search.search(q,new RangeFilter("id",minIP,medIP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,medIP,T,T), numDocs).scoreDocs;
assertEquals("up to med", 1+ medId-minId, result.length); assertEquals("up to med", 1+ medId-minId, result.length);
// unbounded id // unbounded id
result = search.search(q,new RangeFilter("id",minIP,null,T,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,null,T,F), numDocs).scoreDocs;
assertEquals("min and up", numDocs, result.length); assertEquals("min and up", numDocs, result.length);
result = search.search(q,new RangeFilter("id",null,maxIP,F,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",null,maxIP,F,T), numDocs).scoreDocs;
assertEquals("max and down", numDocs, result.length); assertEquals("max and down", numDocs, result.length);
result = search.search(q,new RangeFilter("id",minIP,null,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,null,F,F), numDocs).scoreDocs;
assertEquals("not min, but up", numDocs-1, result.length); assertEquals("not min, but up", numDocs-1, result.length);
result = search.search(q,new RangeFilter("id",null,maxIP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",null,maxIP,F,F), numDocs).scoreDocs;
assertEquals("not max, but down", numDocs-1, result.length); assertEquals("not max, but down", numDocs-1, result.length);
result = search.search(q,new RangeFilter("id",medIP,maxIP,T,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",medIP,maxIP,T,F), numDocs).scoreDocs;
assertEquals("med and up, not max", maxId-medId, result.length); assertEquals("med and up, not max", maxId-medId, result.length);
result = search.search(q,new RangeFilter("id",minIP,medIP,F,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,medIP,F,T), numDocs).scoreDocs;
assertEquals("not min, up to med", medId-minId, result.length); assertEquals("not min, up to med", medId-minId, result.length);
// very small sets // very small sets
result = search.search(q,new RangeFilter("id",minIP,minIP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,minIP,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length); assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,new RangeFilter("id",medIP,medIP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",medIP,medIP,F,F), numDocs).scoreDocs;
assertEquals("med,med,F,F", 0, result.length); assertEquals("med,med,F,F", 0, result.length);
result = search.search(q,new RangeFilter("id",maxIP,maxIP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",maxIP,maxIP,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length); assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,new RangeFilter("id",minIP,minIP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",minIP,minIP,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length); assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,new RangeFilter("id",null,minIP,F,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",null,minIP,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length); assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,new RangeFilter("id",maxIP,maxIP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",maxIP,maxIP,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length); assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,new RangeFilter("id",maxIP,null,T,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",maxIP,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length); assertEquals("max,nul,T,T", 1, result.length);
result = search.search(q,new RangeFilter("id",medIP,medIP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("id",medIP,medIP,T,T), numDocs).scoreDocs;
assertEquals("med,med,T,T", 1, result.length); assertEquals("med,med,T,T", 1, result.length);
} }
public void testRangeFilterIdCollating() throws IOException { public void testFieldCacheRangeFilterRand() throws IOException {
IndexReader reader = IndexReader.open(signedIndex.index); IndexReader reader = IndexReader.open(signedIndex.index);
IndexSearcher search = new IndexSearcher(reader); IndexSearcher search = new IndexSearcher(reader);
Collator c = Collator.getInstance(Locale.ENGLISH); String minRP = pad(signedIndex.minR);
String maxRP = pad(signedIndex.maxR);
int medId = ((maxId - minId) / 2);
String minIP = pad(minId);
String maxIP = pad(maxId);
String medIP = pad(medId);
int numDocs = reader.numDocs();
assertEquals("num of docs", numDocs, 1+ maxId - minId);
Hits result;
Query q = new TermQuery(new Term("body","body"));
// test id, bounded on both ends
result = search.search(q,new RangeFilter("id",minIP,maxIP,T,T,c));
assertEquals("find all", numDocs, result.length());
result = search.search(q,new RangeFilter("id",minIP,maxIP,T,F,c));
assertEquals("all but last", numDocs-1, result.length());
result = search.search(q,new RangeFilter("id",minIP,maxIP,F,T,c));
assertEquals("all but first", numDocs-1, result.length());
result = search.search(q,new RangeFilter("id",minIP,maxIP,F,F,c));
assertEquals("all but ends", numDocs-2, result.length());
result = search.search(q,new RangeFilter("id",medIP,maxIP,T,T,c));
assertEquals("med and up", 1+ maxId-medId, result.length());
result = search.search(q,new RangeFilter("id",minIP,medIP,T,T,c));
assertEquals("up to med", 1+ medId-minId, result.length());
// unbounded id
result = search.search(q,new RangeFilter("id",minIP,null,T,F,c));
assertEquals("min and up", numDocs, result.length());
result = search.search(q,new RangeFilter("id",null,maxIP,F,T,c));
assertEquals("max and down", numDocs, result.length());
result = search.search(q,new RangeFilter("id",minIP,null,F,F,c));
assertEquals("not min, but up", numDocs-1, result.length());
result = search.search(q,new RangeFilter("id",null,maxIP,F,F,c));
assertEquals("not max, but down", numDocs-1, result.length());
result = search.search(q,new RangeFilter("id",medIP,maxIP,T,F,c));
assertEquals("med and up, not max", maxId-medId, result.length());
result = search.search(q,new RangeFilter("id",minIP,medIP,F,T,c));
assertEquals("not min, up to med", medId-minId, result.length());
// very small sets
result = search.search(q,new RangeFilter("id",minIP,minIP,F,F,c));
assertEquals("min,min,F,F", 0, result.length());
result = search.search(q,new RangeFilter("id",medIP,medIP,F,F,c));
assertEquals("med,med,F,F", 0, result.length());
result = search.search(q,new RangeFilter("id",maxIP,maxIP,F,F,c));
assertEquals("max,max,F,F", 0, result.length());
result = search.search(q,new RangeFilter("id",minIP,minIP,T,T,c));
assertEquals("min,min,T,T", 1, result.length());
result = search.search(q,new RangeFilter("id",null,minIP,F,T,c));
assertEquals("nul,min,F,T", 1, result.length());
result = search.search(q,new RangeFilter("id",maxIP,maxIP,T,T,c));
assertEquals("max,max,T,T", 1, result.length());
result = search.search(q,new RangeFilter("id",maxIP,null,T,F,c));
assertEquals("max,nul,T,T", 1, result.length());
result = search.search(q,new RangeFilter("id",medIP,medIP,T,T,c));
assertEquals("med,med,T,T", 1, result.length());
}
public void testRangeFilterRand() throws IOException {
IndexReader reader = IndexReader.open(signedIndex.index);
IndexSearcher search = new IndexSearcher(reader);
String minRP = pad(signedIndex.minR);
String maxRP = pad(signedIndex.maxR);
int numDocs = reader.numDocs(); int numDocs = reader.numDocs();
assertEquals("num of docs", numDocs, 1+ maxId - minId); assertEquals("num of docs", numDocs, 1+ maxId - minId);
ScoreDoc[] result; ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body")); Query q = new TermQuery(new Term("body","body"));
// test extremes, bounded on both ends // test extremes, bounded on both ends
result = search.search(q,new RangeFilter("rand",minRP,maxRP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,maxRP,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length); assertEquals("find all", numDocs, result.length);
result = search.search(q,new RangeFilter("rand",minRP,maxRP,T,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,maxRP,T,F), numDocs).scoreDocs;
assertEquals("all but biggest", numDocs-1, result.length); assertEquals("all but biggest", numDocs-1, result.length);
result = search.search(q,new RangeFilter("rand",minRP,maxRP,F,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,maxRP,F,T), numDocs).scoreDocs;
assertEquals("all but smallest", numDocs-1, result.length); assertEquals("all but smallest", numDocs-1, result.length);
result = search.search(q,new RangeFilter("rand",minRP,maxRP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,maxRP,F,F), numDocs).scoreDocs;
assertEquals("all but extremes", numDocs-2, result.length); assertEquals("all but extremes", numDocs-2, result.length);
// unbounded // unbounded
result = search.search(q,new RangeFilter("rand",minRP,null,T,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,null,T,F), numDocs).scoreDocs;
assertEquals("smallest and up", numDocs, result.length); assertEquals("smallest and up", numDocs, result.length);
result = search.search(q,new RangeFilter("rand",null,maxRP,F,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",null,maxRP,F,T), numDocs).scoreDocs;
assertEquals("biggest and down", numDocs, result.length); assertEquals("biggest and down", numDocs, result.length);
result = search.search(q,new RangeFilter("rand",minRP,null,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,null,F,F), numDocs).scoreDocs;
assertEquals("not smallest, but up", numDocs-1, result.length); assertEquals("not smallest, but up", numDocs-1, result.length);
result = search.search(q,new RangeFilter("rand",null,maxRP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",null,maxRP,F,F), numDocs).scoreDocs;
assertEquals("not biggest, but down", numDocs-1, result.length); assertEquals("not biggest, but down", numDocs-1, result.length);
// very small sets // very small sets
result = search.search(q,new RangeFilter("rand",minRP,minRP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,minRP,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length); assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,new RangeFilter("rand",maxRP,maxRP,F,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",maxRP,maxRP,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length); assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,new RangeFilter("rand",minRP,minRP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",minRP,minRP,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length); assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,new RangeFilter("rand",null,minRP,F,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",null,minRP,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length); assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,new RangeFilter("rand",maxRP,maxRP,T,T), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",maxRP,maxRP,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length); assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,new RangeFilter("rand",maxRP,null,T,F), numDocs).scoreDocs; result = search.search(q,new FieldCacheRangeFilter("rand",maxRP,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length); assertEquals("max,nul,T,T", 1, result.length);
}
}
public void testRangeFilterRandCollating() throws IOException {
// using the unsigned index because collation seems to ignore hyphens
IndexReader reader = IndexReader.open(unsignedIndex.index);
IndexSearcher search = new IndexSearcher(reader);
Collator c = Collator.getInstance(Locale.ENGLISH);
String minRP = pad(unsignedIndex.minR);
String maxRP = pad(unsignedIndex.maxR);
int numDocs = reader.numDocs();
assertEquals("num of docs", numDocs, 1+ maxId - minId);
Hits result;
Query q = new TermQuery(new Term("body","body"));
// test extremes, bounded on both ends
result = search.search(q,new RangeFilter("rand",minRP,maxRP,T,T,c));
assertEquals("find all", numDocs, result.length());
result = search.search(q,new RangeFilter("rand",minRP,maxRP,T,F,c));
assertEquals("all but biggest", numDocs-1, result.length());
result = search.search(q,new RangeFilter("rand",minRP,maxRP,F,T,c));
assertEquals("all but smallest", numDocs-1, result.length());
result = search.search(q,new RangeFilter("rand",minRP,maxRP,F,F,c));
assertEquals("all but extremes", numDocs-2, result.length());
// unbounded
result = search.search(q,new RangeFilter("rand",minRP,null,T,F,c));
assertEquals("smallest and up", numDocs, result.length());
result = search.search(q,new RangeFilter("rand",null,maxRP,F,T,c));
assertEquals("biggest and down", numDocs, result.length());
result = search.search(q,new RangeFilter("rand",minRP,null,F,F,c));
assertEquals("not smallest, but up", numDocs-1, result.length());
result = search.search(q,new RangeFilter("rand",null,maxRP,F,F,c));
assertEquals("not biggest, but down", numDocs-1, result.length());
// very small sets
result = search.search(q,new RangeFilter("rand",minRP,minRP,F,F,c));
assertEquals("min,min,F,F", 0, result.length());
result = search.search(q,new RangeFilter("rand",maxRP,maxRP,F,F,c));
assertEquals("max,max,F,F", 0, result.length());
result = search.search(q,new RangeFilter("rand",minRP,minRP,T,T,c));
assertEquals("min,min,T,T", 1, result.length());
result = search.search(q,new RangeFilter("rand",null,minRP,F,T,c));
assertEquals("nul,min,F,T", 1, result.length());
result = search.search(q,new RangeFilter("rand",maxRP,maxRP,T,T,c));
assertEquals("max,max,T,T", 1, result.length());
result = search.search(q,new RangeFilter("rand",maxRP,null,T,F,c));
assertEquals("max,nul,T,T", 1, result.length());
}
public void testFarsi() throws Exception {
/* build an index */
RAMDirectory farsiIndex = new RAMDirectory();
IndexWriter writer = new IndexWriter(farsiIndex, new SimpleAnalyzer(), T,
IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
doc.add(new Field("content","\u0633\u0627\u0628",
Field.Store.YES, Field.Index.UN_TOKENIZED));
doc.add(new Field("body", "body",
Field.Store.YES, Field.Index.UN_TOKENIZED));
writer.addDocument(doc);
writer.optimize();
writer.close();
IndexReader reader = IndexReader.open(farsiIndex);
IndexSearcher search = new IndexSearcher(reader);
Query q = new TermQuery(new Term("body","body"));
// Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
// RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
// characters properly.
Collator collator = Collator.getInstance(new Locale("ar"));
// Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
// orders the U+0698 character before the U+0633 character, so the single
// index Term below should NOT be returned by a RangeFilter with a Farsi
// Collator (or an Arabic one for the case when Farsi is not supported).
Hits result = search.search
(q, new RangeFilter("content", "\u062F", "\u0698", T, T, collator));
assertEquals("The index Term should not be included.", 0, result.length());
result = search.search
(q, new RangeFilter("content", "\u0633", "\u0638", T, T, collator));
assertEquals("The index Term should be included.", 1, result.length());
search.close();
}
} }