Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-09 11:35:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

add comment and test for ngram token filter

Browse Source
This commit is contained in:
Nathan Gass 2017-01-13 16:42:41 +01:00
parent 01f2a87c67
commit 80e2854247
1 changed files with 25 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
View File

@ -79,6 +79,28 @@ public class TestNGramFilters extends BaseTokenStreamFactoryTestCase {
new String[] { "te", "tes", "es", "est", "st" }); new String[] { "te", "tes", "es", "est", "st" });
} }
/**
* Test NGramFilterFactory on tokens with payloads
*/
public void testNGramFilterPayload() throws Exception {
Reader reader = new StringReader("test|0.1");
TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);
stream = tokenFilterFactory("NGram", "minGramSize", "1", "maxGramSize", "2").create(stream);
stream.reset();
while (stream.incrementToken()) {
PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
assertNotNull(payAttr);
BytesRef payData = payAttr.getPayload();
assertNotNull(payData);
float payFloat = PayloadHelper.decodeFloat(payData.bytes);
assertEquals(0.1f, payFloat, 0.0f);
}
stream.end();
stream.close();
}
/** /**
* Test EdgeNGramTokenizerFactory * Test EdgeNGramTokenizerFactory
*/ */
@ -127,6 +149,9 @@ public class TestNGramFilters extends BaseTokenStreamFactoryTestCase {
new String[] { "t", "te" }); new String[] { "t", "te" });
} }
/**
* Test EdgeNGramFilterFactory on tokens with payloads
*/
public void testEdgeNGramFilterPayload() throws Exception { public void testEdgeNGramFilterPayload() throws Exception {
Reader reader = new StringReader("test|0.1"); Reader reader = new StringReader("test|0.1");
TokenStream stream = whitespaceMockTokenizer(reader); TokenStream stream = whitespaceMockTokenizer(reader);