Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3717: add tests 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1235199 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-01-24 10:40:46 +00:00
parent 0aad3d64ad
commit 6edfe4f157
8 changed files with 146 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
View File

@ -30,7 +30,7 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
public final class PositionFilter extends TokenFilter {
/** Position increment to assign to all but the first token - default = 0 */
private int positionIncrement = 0;
private final int positionIncrement;
/** The first token must have non-zero positionIncrement **/
private boolean firstTokenPositioned = false;
@ -44,7 +44,7 @@ public final class PositionFilter extends TokenFilter {
* @param input the input stream
*/
public PositionFilter(final TokenStream input) {
super(input);
this(input, 0);
}
/**
@ -56,7 +56,7 @@ public final class PositionFilter extends TokenFilter {
* token from the input stream
*/
public PositionFilter(final TokenStream input, final int positionIncrement) {
this(input);
super(input);
this.positionIncrement = positionIncrement;
}

15
modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
View File

@ -17,10 +17,13 @@ package org.apache.lucene.analysis.path;
* limitations under the License.
*/
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
@ -193,4 +196,16 @@ public class TestPathHierarchyTokenizer extends BaseTokenStreamTestCase {
new int[]{1},
path.length());
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
}
}

16
modules/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
View File

@ -17,9 +17,13 @@ package org.apache.lucene.analysis.path;
* limitations under the License.
*/
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
public class TestReversePathHierarchyTokenizer extends BaseTokenStreamTestCase {
@ -154,4 +158,16 @@ public class TestReversePathHierarchyTokenizer extends BaseTokenStreamTestCase {
new int[]{1, 0},
path.length());
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new ReversePathHierarchyTokenizer(reader);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
}
}

20
modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java
View File

@ -18,14 +18,17 @@
package org.apache.lucene.analysis.pattern;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
/**
* Tests {@link PatternReplaceCharFilter}
@ -172,4 +175,21 @@ public class TestPatternReplaceCharFilter extends BaseTokenStreamTestCase {
private Pattern pattern( String p ){
return Pattern.compile( p );
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
@Override
protected Reader initReader(Reader reader) {
return new PatternReplaceCharFilter(Pattern.compile("a"), "b", CharReader.get(reader));
}
};
checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
}
}

26
modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
View File

@ -17,10 +17,13 @@
package org.apache.lucene.analysis.pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import java.io.Reader;
import java.io.StringReader;
import java.util.regex.Pattern;
@ -77,5 +80,28 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase {
assertTokenStreamContents(ts,
new String[] { "aa$fooaa$fooa$foo$", "a$", "caaaaaaaaa$" });
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false);
return new TokenStreamComponents(tokenizer, filter);
}
};
checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
Analyzer b = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true);
return new TokenStreamComponents(tokenizer, filter);
}
};
checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
}
}

36
modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
View File

@ -18,17 +18,22 @@
package org.apache.lucene.analysis.pattern;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
import org.apache.lucene.analysis.path.PathHierarchyTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class TestPatternTokenizer extends BaseTokenStreamTestCase
@ -117,4 +122,35 @@ public class TestPatternTokenizer extends BaseTokenStreamTestCase
in.close();
return out.toString();
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = null;
try {
tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), -1);
} catch (IOException e) {
throw new RuntimeException(e);
}
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
Analyzer b = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = null;
try {
tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), 0);
} catch (IOException e) {
throw new RuntimeException(e);
}
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
}
}

15
modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
View File

@ -17,11 +17,14 @@
package org.apache.lucene.analysis.reverse;
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.util.Version;
public class TestReverseStringFilter extends BaseTokenStreamTestCase {
@ -96,4 +99,16 @@ public class TestReverseStringFilter extends BaseTokenStreamTestCase {
ReverseStringFilter.reverse(TEST_VERSION_CURRENT, buffer, 3, 7);
assertEquals("abcfed𩬅愯瀛", new String(buffer));
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
}
}

15
modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
View File

@ -18,9 +18,12 @@ package org.apache.lucene.analysis.shingle;
*/
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@ -1129,4 +1132,16 @@ public class ShingleFilterTest extends BaseTokenStreamTestCase {
token.setPositionIncrement(positionIncrement);
return token;
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
}
};
checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
}
}