From 60c3278f9ab0bfb3a0d47317e59fed99526ce560 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Tue, 12 Oct 2010 09:55:29 +0000 Subject: [PATCH] LUCENE-2696: MockAnalyzer injects payloads for better test coverage git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1021720 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/TestExternalCodecs.java | 19 ++++++++----- .../apache/lucene/analysis/MockAnalyzer.java | 13 ++++++--- .../apache/lucene/index/TestIndexWriter.java | 3 +-- .../lucene/index/TestLazyProxSkipping.java | 3 ++- .../lucene/search/spans/TestBasics.java | 27 +++++++++---------- 5 files changed, 38 insertions(+), 27 deletions(-) diff --git a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java index 3277df0b2f3..2d421b03808 100644 --- a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java +++ b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java @@ -129,6 +129,8 @@ public class TestExternalCodecs extends LuceneTestCase { static class RAMDoc { final int docID; final int[] positions; + byte[][] payloads; + public RAMDoc(int docID, int freq) { this.docID = docID; positions = new int[freq]; @@ -212,10 +214,15 @@ public class TestExternalCodecs extends LuceneTestCase { @Override public void addPosition(int position, BytesRef payload) { - if (payload != null) { - throw new UnsupportedOperationException("can't handle payloads"); + current.positions[posUpto] = position; + if (payload != null && payload.length > 0) { + if (current.payloads == null) { + current.payloads = new byte[current.positions.length][]; + } + byte[] bytes = current.payloads[posUpto] = new byte[payload.length]; + System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length); } - current.positions[posUpto++] = position; + posUpto++; } @Override @@ -436,12 +443,12 @@ public class TestExternalCodecs extends LuceneTestCase { @Override public boolean hasPayload() { - return false; + return current.payloads != null && current.payloads[posUpto-1] != null; } @Override public BytesRef getPayload() { - return null; + return new BytesRef(current.payloads[posUpto-1]); } } @@ -614,7 +621,7 @@ public class TestExternalCodecs extends LuceneTestCase { final int NUM_DOCS = 173; Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setCodecProvider(new MyCodecs())); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, true)).setCodecProvider(new MyCodecs())); w.setMergeFactor(3); Document doc = new Document(); diff --git a/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java b/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java index 18cdad0b934..d23b093031f 100644 --- a/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java +++ b/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java @@ -22,7 +22,6 @@ import java.io.Reader; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Payload; import org.apache.lucene.util.automaton.CharacterRunAutomaton; @@ -38,7 +37,7 @@ public final class MockAnalyzer extends Analyzer { private int positionIncrementGap; public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) { - this(runAutomaton, lowerCase, filter, enablePositionIncrements, false); + this(runAutomaton, lowerCase, filter, enablePositionIncrements, true); } /** @@ -65,7 +64,7 @@ public final class MockAnalyzer extends Analyzer { * @param lowerCase true if the tokenizer should lowercase terms */ public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) { - this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false, false); + this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false, true); } public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, boolean payload) { @@ -148,4 +147,10 @@ final class SimplePayloadFilter extends TokenFilter { return false; } } -} \ No newline at end of file + + @Override + public void reset() throws IOException { + super.reset(); + pos = 0; + } +} diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index 0dcfd9e8110..fdab74d588f 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -73,7 +73,6 @@ import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.UnicodeUtil; -import org.apache.lucene.util.Version; import org.apache.lucene.util._TestUtil; import org.apache.lucene.util.ThreadInterruptedException; import org.apache.lucene.util.BytesRef; @@ -4962,7 +4961,7 @@ public class TestIndexWriter extends LuceneTestCase { final Random r = random; Directory dir = newDirectory(); - FlushCountingIndexWriter w = new FlushCountingIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setRAMBufferSizeMB(0.5).setMaxBufferedDocs(-1).setMaxBufferedDeleteTerms(-1)); + FlushCountingIndexWriter w = new FlushCountingIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setRAMBufferSizeMB(0.5).setMaxBufferedDocs(-1).setMaxBufferedDeleteTerms(-1)); //w.setInfoStream(System.out); Document doc = new Document(); doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED)); diff --git a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java index bfaac41eafb..d2f2d7d5e39 100755 --- a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java +++ b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.document.Field; @@ -69,7 +70,7 @@ public class TestLazyProxSkipping extends LuceneTestCase { int numDocs = 500; Directory directory = new SeekCountingDirectory(new RAMDirectory()); - IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10)); + IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); for (int i = 0; i < numDocs; i++) { diff --git a/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java b/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java index 9e76db29b4d..3415247f28d 100644 --- a/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java +++ b/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.MockPayloadAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -356,8 +355,8 @@ public class TestBasics extends LuceneTestCase { Payload pay = new Payload(("pos: " + 5).getBytes()); SpanQuery query = new SpanPayloadCheckQuery(term1, Collections.singletonList(pay.getData())); checkHits(query, new int[] - {5}); - assertTrue(searcher.explain(query, 5).getValue() > 0.0f); + {1125, 1135, 1145, 1155, 1165, 1175, 1185, 1195, 1225, 1235, 1245, 1255, 1265, 1275, 1285, 1295, 1325, 1335, 1345, 1355, 1365, 1375, 1385, 1395, 1425, 1435, 1445, 1455, 1465, 1475, 1485, 1495, 1525, 1535, 1545, 1555, 1565, 1575, 1585, 1595, 1625, 1635, 1645, 1655, 1665, 1675, 1685, 1695, 1725, 1735, 1745, 1755, 1765, 1775, 1785, 1795, 1825, 1835, 1845, 1855, 1865, 1875, 1885, 1895, 1925, 1935, 1945, 1955, 1965, 1975, 1985, 1995}); + assertTrue(searcher.explain(query, 1125).getValue() > 0.0f); SpanTermQuery term2 = new SpanTermQuery(new Term("field", "hundred")); SpanNearQuery snq; @@ -368,22 +367,22 @@ public class TestBasics extends LuceneTestCase { clauses[0] = term1; clauses[1] = term2; snq = new SpanNearQuery(clauses, 0, true); - pay = new Payload(("pos: " + 1656).getBytes()); - pay2 = new Payload(("pos: " + 1657).getBytes()); + pay = new Payload(("pos: " + 0).getBytes()); + pay2 = new Payload(("pos: " + 1).getBytes()); list = new ArrayList(); list.add(pay.getData()); list.add(pay2.getData()); query = new SpanNearPayloadCheckQuery(snq, list); checkHits(query, new int[] - {500}); + {500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599}); clauses = new SpanQuery[3]; clauses[0] = term1; clauses[1] = term2; clauses[2] = new SpanTermQuery(new Term("field", "five")); snq = new SpanNearQuery(clauses, 0, true); - pay = new Payload(("pos: " + 1670).getBytes()); - pay2 = new Payload(("pos: " + 1671).getBytes()); - Payload pay3 = new Payload(("pos: " + 1672).getBytes()); + pay = new Payload(("pos: " + 0).getBytes()); + pay2 = new Payload(("pos: " + 1).getBytes()); + Payload pay3 = new Payload(("pos: " + 2).getBytes()); list = new ArrayList(); list.add(pay.getData()); list.add(pay2.getData()); @@ -412,16 +411,16 @@ public class TestBasics extends LuceneTestCase { checkHits(query, new int[]{1103, 1203,1303,1403,1503,1603,1703,1803,1903}); Collection payloads = new ArrayList(); - Payload pay = new Payload(("pos: " + 3896).getBytes()); - Payload pay2 = new Payload(("pos: " + 3897).getBytes()); - Payload pay3 = new Payload(("pos: " + 3899).getBytes()); - Payload pay4 = new Payload(("pos: " + 3900).getBytes()); + Payload pay = new Payload(("pos: " + 0).getBytes()); + Payload pay2 = new Payload(("pos: " + 1).getBytes()); + Payload pay3 = new Payload(("pos: " + 3).getBytes()); + Payload pay4 = new Payload(("pos: " + 4).getBytes()); payloads.add(pay.getData()); payloads.add(pay2.getData()); payloads.add(pay3.getData()); payloads.add(pay4.getData()); query = new SpanNearPayloadCheckQuery(oneThousHunThree, payloads); - checkHits(query, new int[]{1103}); + checkHits(query, new int[]{1103, 1203,1303,1403,1503,1603,1703,1803,1903}); }