Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2696: MockAnalyzer injects payloads for better test coverage 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1021720 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2010-10-12 09:55:29 +00:00
parent 3fc915a069
commit 60c3278f9a
5 changed files with 38 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lucene/src/test/org/apache/lucene/TestExternalCodecs.java
View File

@ -129,6 +129,8 @@ public class TestExternalCodecs extends LuceneTestCase {
static class RAMDoc {
final int docID;
final int[] positions;
byte[][] payloads;
public RAMDoc(int docID, int freq) {
this.docID = docID;
positions = new int[freq];
@ -212,10 +214,15 @@ public class TestExternalCodecs extends LuceneTestCase {
@Override
public void addPosition(int position, BytesRef payload) {
if (payload != null) {
throw new UnsupportedOperationException("can't handle payloads");
current.positions[posUpto] = position;
if (payload != null && payload.length > 0) {
if (current.payloads == null) {
current.payloads = new byte[current.positions.length][];
}
byte[] bytes = current.payloads[posUpto] = new byte[payload.length];
System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length);
}
current.positions[posUpto++] = position;
posUpto++;
}
@Override
@ -436,12 +443,12 @@ public class TestExternalCodecs extends LuceneTestCase {
@Override
public boolean hasPayload() {
return false;
return current.payloads != null && current.payloads[posUpto-1] != null;
}
@Override
public BytesRef getPayload() {
return null;
return new BytesRef(current.payloads[posUpto-1]);
}
}
@ -614,7 +621,7 @@ public class TestExternalCodecs extends LuceneTestCase {
final int NUM_DOCS = 173;
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setCodecProvider(new MyCodecs()));
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, true)).setCodecProvider(new MyCodecs()));
w.setMergeFactor(3);
Document doc = new Document();

13
lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java
View File

@ -22,7 +22,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.index.Payload;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
@ -38,7 +37,7 @@ public final class MockAnalyzer extends Analyzer {
private int positionIncrementGap;
public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
this(runAutomaton, lowerCase, filter, enablePositionIncrements, false);
this(runAutomaton, lowerCase, filter, enablePositionIncrements, true);
}
/**
@ -65,7 +64,7 @@ public final class MockAnalyzer extends Analyzer {
* @param lowerCase true if the tokenizer should lowercase terms
*/
public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {
this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false, false);
this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false, true);
}
public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, boolean payload) {
@ -148,4 +147,10 @@ final class SimplePayloadFilter extends TokenFilter {
return false;
}
}
}
@Override
public void reset() throws IOException {
super.reset();
pos = 0;
}
}

3
lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -73,7 +73,6 @@ import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.BytesRef;
@ -4962,7 +4961,7 @@ public class TestIndexWriter extends LuceneTestCase {
final Random r = random;
Directory dir = newDirectory();
FlushCountingIndexWriter w = new FlushCountingIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setRAMBufferSizeMB(0.5).setMaxBufferedDocs(-1).setMaxBufferedDeleteTerms(-1));
FlushCountingIndexWriter w = new FlushCountingIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setRAMBufferSizeMB(0.5).setMaxBufferedDocs(-1).setMaxBufferedDeleteTerms(-1));
//w.setInfoStream(System.out);
Document doc = new Document();
doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED));

3
lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.document.Field;
@ -69,7 +70,7 @@ public class TestLazyProxSkipping extends LuceneTestCase {
int numDocs = 500;
Directory directory = new SeekCountingDirectory(new RAMDirectory());
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10));
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10));
((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false);
((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false);
for (int i = 0; i < numDocs; i++) {

27
lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
View File

@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockPayloadAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@ -356,8 +355,8 @@ public class TestBasics extends LuceneTestCase {
Payload pay = new Payload(("pos: " + 5).getBytes());
SpanQuery query = new SpanPayloadCheckQuery(term1, Collections.singletonList(pay.getData()));
checkHits(query, new int[]
{5});
assertTrue(searcher.explain(query, 5).getValue() > 0.0f);
{1125, 1135, 1145, 1155, 1165, 1175, 1185, 1195, 1225, 1235, 1245, 1255, 1265, 1275, 1285, 1295, 1325, 1335, 1345, 1355, 1365, 1375, 1385, 1395, 1425, 1435, 1445, 1455, 1465, 1475, 1485, 1495, 1525, 1535, 1545, 1555, 1565, 1575, 1585, 1595, 1625, 1635, 1645, 1655, 1665, 1675, 1685, 1695, 1725, 1735, 1745, 1755, 1765, 1775, 1785, 1795, 1825, 1835, 1845, 1855, 1865, 1875, 1885, 1895, 1925, 1935, 1945, 1955, 1965, 1975, 1985, 1995});
assertTrue(searcher.explain(query, 1125).getValue() > 0.0f);
SpanTermQuery term2 = new SpanTermQuery(new Term("field", "hundred"));
SpanNearQuery snq;
@ -368,22 +367,22 @@ public class TestBasics extends LuceneTestCase {
clauses[0] = term1;
clauses[1] = term2;
snq = new SpanNearQuery(clauses, 0, true);
pay = new Payload(("pos: " + 1656).getBytes());
pay2 = new Payload(("pos: " + 1657).getBytes());
pay = new Payload(("pos: " + 0).getBytes());
pay2 = new Payload(("pos: " + 1).getBytes());
list = new ArrayList<byte[]>();
list.add(pay.getData());
list.add(pay2.getData());
query = new SpanNearPayloadCheckQuery(snq, list);
checkHits(query, new int[]
{500});
{500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599});
clauses = new SpanQuery[3];
clauses[0] = term1;
clauses[1] = term2;
clauses[2] = new SpanTermQuery(new Term("field", "five"));
snq = new SpanNearQuery(clauses, 0, true);
pay = new Payload(("pos: " + 1670).getBytes());
pay2 = new Payload(("pos: " + 1671).getBytes());
Payload pay3 = new Payload(("pos: " + 1672).getBytes());
pay = new Payload(("pos: " + 0).getBytes());
pay2 = new Payload(("pos: " + 1).getBytes());
Payload pay3 = new Payload(("pos: " + 2).getBytes());
list = new ArrayList<byte[]>();
list.add(pay.getData());
list.add(pay2.getData());
@ -412,16 +411,16 @@ public class TestBasics extends LuceneTestCase {
checkHits(query, new int[]{1103, 1203,1303,1403,1503,1603,1703,1803,1903});
Collection<byte[]> payloads = new ArrayList<byte[]>();
Payload pay = new Payload(("pos: " + 3896).getBytes());
Payload pay2 = new Payload(("pos: " + 3897).getBytes());
Payload pay3 = new Payload(("pos: " + 3899).getBytes());
Payload pay4 = new Payload(("pos: " + 3900).getBytes());
Payload pay = new Payload(("pos: " + 0).getBytes());
Payload pay2 = new Payload(("pos: " + 1).getBytes());
Payload pay3 = new Payload(("pos: " + 3).getBytes());
Payload pay4 = new Payload(("pos: " + 4).getBytes());
payloads.add(pay.getData());
payloads.add(pay2.getData());
payloads.add(pay3.getData());
payloads.add(pay4.getData());
query = new SpanNearPayloadCheckQuery(oneThousHunThree, payloads);
checkHits(query, new int[]{1103});
checkHits(query, new int[]{1103, 1203,1303,1403,1503,1603,1703,1803,1903});
}