Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2372: switch over remaining uses of TermAttribute 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@950008 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2010-06-01 10:35:13 +00:00
parent 7661fe8c04
commit ad0e495911
116 changed files with 822 additions and 1433 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
View File

@ -26,9 +26,10 @@ import java.util.List;
import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.benchmark.BenchmarkTestCase;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker;
@ -918,11 +919,11 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
TokenStream ts2 = a2.tokenStream("bogus", new StringReader(text));
ts1.reset();
ts2.reset();
TermAttribute termAtt1 = ts1.addAttribute(TermAttribute.class);
TermAttribute termAtt2 = ts2.addAttribute(TermAttribute.class);
CharTermAttribute termAtt1 = ts1.addAttribute(CharTermAttribute.class);
CharTermAttribute termAtt2 = ts2.addAttribute(CharTermAttribute.class);
assertTrue(ts1.incrementToken());
assertTrue(ts2.incrementToken());
assertEquals(termAtt1.term(), termAtt2.term());
assertEquals(termAtt1.toString(), termAtt2.toString());
assertFalse(ts1.incrementToken());
assertFalse(ts2.incrementToken());
ts1.close();
@ -994,21 +995,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
private void assertEqualShingle
(Analyzer analyzer, String text, String[] expected) throws Exception {
TokenStream stream = analyzer.tokenStream("bogus", new StringReader(text));
stream.reset();
TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
int termNum = 0;
while (stream.incrementToken()) {
assertTrue("Extra output term(s), starting with '"
+ new String(termAtt.termBuffer(), 0, termAtt.termLength()) + "'",
termNum < expected.length);
assertEquals("Mismatch in output term # " + termNum + " - ",
expected[termNum],
new String(termAtt.termBuffer(), 0, termAtt.termLength()));
++termNum;
}
assertEquals("Too few output terms", expected.length, termNum);
stream.close();
BaseTokenStreamTestCase.assertAnalyzesTo(analyzer, text, expected);
}
private String[] getShingleConfig(String params) {

6
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
View File

@ -23,9 +23,9 @@ import java.util.Iterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.PriorityQueue;
/**
@ -191,7 +191,7 @@ public class Highlighter
ArrayList<TextFragment> docFrags = new ArrayList<TextFragment>();
StringBuilder newText=new StringBuilder();
TermAttribute termAtt = tokenStream.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
tokenStream.addAttribute(PositionIncrementAttribute.class);
tokenStream.reset();
@ -225,7 +225,7 @@ public class Highlighter
(offsetAtt.startOffset()>text.length())
)
{
throw new InvalidTokenOffsetsException("Token "+ termAtt.term()
throw new InvalidTokenOffsetsException("Token "+ termAtt.toString()
+" exceeds length of provided text sized "+text.length());
}
if((tokenGroup.numTokens>0)&&(tokenGroup.isDistinct()))

8
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
View File

@ -25,8 +25,8 @@ import java.util.Set;
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.memory.MemoryIndex;
import org.apache.lucene.search.Query;
@ -46,7 +46,7 @@ public class QueryScorer implements Scorer {
private float maxTermWeight;
private int position = -1;
private String defaultField;
private TermAttribute termAtt;
private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncAtt;
private boolean expandMultiTermQuery = true;
private Query query;
@ -145,7 +145,7 @@ public class QueryScorer implements Scorer {
*/
public float getTokenScore() {
position += posIncAtt.getPositionIncrement();
String termText = termAtt.term();
String termText = termAtt.toString();
WeightedSpanTerm weightedSpanTerm;
@ -175,7 +175,7 @@ public class QueryScorer implements Scorer {
*/
public TokenStream init(TokenStream tokenStream) throws IOException {
position = -1;
termAtt = tokenStream.addAttribute(TermAttribute.class);
termAtt = tokenStream.addAttribute(CharTermAttribute.class);
posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
if(!skipInitExtractor) {
if(fieldWeightedSpanTerms != null) {

8
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
View File

@ -21,7 +21,7 @@ import java.util.HashMap;
import java.util.HashSet;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
@ -41,7 +41,7 @@ public class QueryTermScorer implements Scorer {
float maxTermWeight = 0;
private HashMap<String,WeightedTerm> termsToFind;
private TermAttribute termAtt;
private CharTermAttribute termAtt;
/**
*
@ -95,7 +95,7 @@ public class QueryTermScorer implements Scorer {
* @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
*/
public TokenStream init(TokenStream tokenStream) {
termAtt = tokenStream.addAttribute(TermAttribute.class);
termAtt = tokenStream.addAttribute(CharTermAttribute.class);
return null;
}
@ -118,7 +118,7 @@ public class QueryTermScorer implements Scorer {
* @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
*/
public float getTokenScore() {
String termText = termAtt.term();
String termText = termAtt.toString();
WeightedTerm queryTerm = termsToFind.get(termText);
if (queryTerm == null) {

8
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
View File

@ -20,9 +20,9 @@ package org.apache.lucene.search.highlight;
import java.util.List;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.search.spans.Spans;
@ -38,7 +38,7 @@ public class SimpleSpanFragmenter implements Fragmenter {
private QueryScorer queryScorer;
private int waitForPos = -1;
private int textSize;
private TermAttribute termAtt;
private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncAtt;
private OffsetAttribute offsetAtt;
@ -70,7 +70,7 @@ public class SimpleSpanFragmenter implements Fragmenter {
return false;
}
WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.term());
WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.toString());
if (wSpanTerm != null) {
List<PositionSpan> positionSpans = wSpanTerm.getPositionSpans();
@ -101,7 +101,7 @@ public class SimpleSpanFragmenter implements Fragmenter {
position = -1;
currentNumFrags = 1;
textSize = originalText.length();
termAtt = tokenStream.addAttribute(TermAttribute.class);
termAtt = tokenStream.addAttribute(CharTermAttribute.class);
posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
}

8
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
View File

@ -19,8 +19,8 @@ package org.apache.lucene.search.highlight;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
/**
* One, or several overlapping tokens, along with the score(s) and the scope of
@ -38,11 +38,11 @@ public class TokenGroup {
int matchStartOffset, matchEndOffset;
private OffsetAttribute offsetAtt;
private TermAttribute termAtt;
private CharTermAttribute termAtt;
public TokenGroup(TokenStream tokenStream) {
offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
termAtt = tokenStream.addAttribute(TermAttribute.class);
termAtt = tokenStream.addAttribute(CharTermAttribute.class);
}
void addToken(float score) {
@ -68,7 +68,7 @@ public class TokenGroup {
}
}
Token token = new Token(termStartOffset, termEndOffset);
token.setTermBuffer(termAtt.term());
token.setEmpty().append(termAtt);
tokens[numTokens] = token;
scores[numTokens] = score;
numTokens++;

11
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
View File

@ -29,8 +29,8 @@ import java.util.Comparator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermFreqVector;
@ -153,13 +153,13 @@ public class TokenSources {
int currentToken = 0;
TermAttribute termAtt;
CharTermAttribute termAtt;
OffsetAttribute offsetAtt;
StoredTokenStream(Token tokens[]) {
this.tokens = tokens;
termAtt = addAttribute(TermAttribute.class);
termAtt = addAttribute(CharTermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
@ -170,7 +170,7 @@ public class TokenSources {
}
Token token = tokens[currentToken++];
clearAttributes();
termAtt.setTermBuffer(token.term());
termAtt.setEmpty().append(token);
offsetAtt.setOffset(token.startOffset(), token.endOffset());
return true;
}
@ -204,9 +204,8 @@ public class TokenSources {
unsortedTokens = new ArrayList<Token>();
}
for (int tp = 0; tp < offsets.length; tp++) {
Token token = new Token(offsets[tp].getStartOffset(), offsets[tp]
Token token = new Token(terms[t], offsets[tp].getStartOffset(), offsets[tp]
.getEndOffset());
token.setTermBuffer(terms[t]);
unsortedTokens.add(token);
}
} else {

10
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
View File

@ -25,9 +25,9 @@ import java.util.List;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorOffsetInfo;
@ -37,7 +37,7 @@ public final class TokenStreamFromTermPositionVector extends TokenStream {
private Iterator<Token> tokensAtCurrentPosition;
private TermAttribute termAttribute;
private CharTermAttribute termAttribute;
private PositionIncrementAttribute positionIncrementAttribute;
@ -51,7 +51,7 @@ public final class TokenStreamFromTermPositionVector extends TokenStream {
*/
public TokenStreamFromTermPositionVector(
final TermPositionVector termPositionVector) {
termAttribute = addAttribute(TermAttribute.class);
termAttribute = addAttribute(CharTermAttribute.class);
positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
offsetAttribute = addAttribute(OffsetAttribute.class);
final String[] terms = termPositionVector.getTerms();
@ -65,7 +65,7 @@ public final class TokenStreamFromTermPositionVector extends TokenStream {
offsets[j].getStartOffset(), offsets[j].getEndOffset());
} else {
token = new Token();
token.setTermBuffer(terms[i]);
token.setEmpty().append(terms[i]);
}
// Yes - this is the position, not the increment! This is for
// sorting. This value
@ -100,7 +100,7 @@ public final class TokenStreamFromTermPositionVector extends TokenStream {
if (this.tokensAtCurrentPosition.hasNext()) {
final Token next = this.tokensAtCurrentPosition.next();
clearAttributes();
termAttribute.setTermBuffer(next.term());
termAttribute.setEmpty().append(next);
positionIncrementAttribute.setPositionIncrement(next
.getPositionIncrement());
offsetAttribute.setOffset(next.startOffset(), next.endOffset());

30
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
@ -296,16 +296,11 @@ public class HighlighterPhraseTest extends LuceneTestCase {
private int i = -1;
private TermAttribute termAttribute;
private OffsetAttribute offsetAttribute;
private PositionIncrementAttribute positionIncrementAttribute;
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
public TokenStreamSparse() {
termAttribute = addAttribute(TermAttribute.class);
offsetAttribute = addAttribute(OffsetAttribute.class);
positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
reset();
}
@ -316,8 +311,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
return false;
}
clearAttributes();
termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
.term().length());
termAttribute.setEmpty().append(this.tokens[i]);
offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
.endOffset());
positionIncrementAttribute.setPositionIncrement(this.tokens[i]
@ -342,16 +336,11 @@ public class HighlighterPhraseTest extends LuceneTestCase {
private int i = -1;
private TermAttribute termAttribute;
private OffsetAttribute offsetAttribute;
private PositionIncrementAttribute positionIncrementAttribute;
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
public TokenStreamConcurrent() {
termAttribute = addAttribute(TermAttribute.class);
offsetAttribute = addAttribute(OffsetAttribute.class);
positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
reset();
}
@ -362,8 +351,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
return false;
}
clearAttributes();
termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
.term().length());
termAttribute.setEmpty().append(this.tokens[i]);
offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
.endOffset());
positionIncrementAttribute.setPositionIncrement(this.tokens[i]

46
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
View File

@ -41,7 +41,7 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericField;
@ -1424,13 +1424,10 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
return new TokenStream() {
Iterator<Token> iter;
List<Token> lst;
private TermAttribute termAtt;
private PositionIncrementAttribute posIncrAtt;
private OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
{
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
lst = new ArrayList<Token>();
Token t;
t = createToken("hi", 0, 2);
@ -1456,7 +1453,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
if(iter.hasNext()) {
Token token = iter.next();
clearAttributes();
termAtt.setTermBuffer(token.term());
termAtt.setEmpty().append(token);
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
return true;
@ -1473,13 +1470,10 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
return new TokenStream() {
Iterator<Token> iter;
List<Token> lst;
private TermAttribute termAtt;
private PositionIncrementAttribute posIncrAtt;
private OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
{
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
lst = new ArrayList<Token>();
Token t;
t = createToken("hispeed", 0, 8);
@ -1505,7 +1499,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
if(iter.hasNext()) {
Token token = iter.next();
clearAttributes();
termAtt.setTermBuffer(token.term());
termAtt.setEmpty().append(token);
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
return true;
@ -1762,9 +1756,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
private static Token createToken(String term, int start, int offset)
{
Token token = new Token(start, offset);
token.setTermBuffer(term);
return token;
return new Token(term, start, offset);
}
}
@ -1795,7 +1787,7 @@ final class SynonymAnalyzer extends Analyzer {
@Override
public TokenStream tokenStream(String arg0, Reader arg1) {
Tokenizer stream = new MockTokenizer(arg1, MockTokenizer.SIMPLE, true);
stream.addAttribute(TermAttribute.class);
stream.addAttribute(CharTermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
return new SynonymTokenizer(stream, synonyms);
@ -1811,21 +1803,21 @@ final class SynonymTokenizer extends TokenStream {
private Token currentRealToken = null;
private Map<String,String> synonyms;
StringTokenizer st = null;
private TermAttribute realTermAtt;
private CharTermAttribute realTermAtt;
private PositionIncrementAttribute realPosIncrAtt;
private OffsetAttribute realOffsetAtt;
private TermAttribute termAtt;
private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncrAtt;
private OffsetAttribute offsetAtt;
public SynonymTokenizer(TokenStream realStream, Map<String,String> synonyms) {
this.realStream = realStream;
this.synonyms = synonyms;
realTermAtt = realStream.addAttribute(TermAttribute.class);
realTermAtt = realStream.addAttribute(CharTermAttribute.class);
realPosIncrAtt = realStream.addAttribute(PositionIncrementAttribute.class);
realOffsetAtt = realStream.addAttribute(OffsetAttribute.class);
termAtt = addAttribute(TermAttribute.class);
termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
@ -1840,25 +1832,25 @@ final class SynonymTokenizer extends TokenStream {
}
//Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());
clearAttributes();
termAtt.setTermBuffer(realTermAtt.term());
termAtt.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement());
String expansions = synonyms.get(realTermAtt.term());
String expansions = synonyms.get(realTermAtt.toString());
if (expansions == null) {
return true;
}
st = new StringTokenizer(expansions, ",");
if (st.hasMoreTokens()) {
currentRealToken = new Token(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
currentRealToken.setTermBuffer(realTermAtt.term());
currentRealToken.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
}
return true;
} else {
String tok = st.nextToken();
clearAttributes();
termAtt.setTermBuffer(tok);
termAtt.setEmpty().append(tok);
offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset());
posIncrAtt.setPositionIncrement(0);
if (!st.hasMoreTokens()) {

6
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
View File

@ -26,8 +26,8 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
@ -221,14 +221,14 @@ public abstract class AbstractTestCase extends LuceneTestCase {
ch = 0;
}
TermAttribute termAtt = addAttribute(TermAttribute.class);
CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
public boolean incrementToken() throws IOException {
if( !getNextPartialSnippet() )
return false;
clearAttributes();
termAtt.setTermBuffer(snippet, startTerm, lenTerm);
termAtt.setEmpty().append(snippet, startTerm, startTerm + lenTerm);
offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));
return true;
}

4
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
View File

@ -25,7 +25,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.AttributeImpl;
@ -301,7 +301,7 @@ public class IndexTimeSynonymTest extends AbstractTestCase {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class);
final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
int p = 0;
@Override

6
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
View File

@ -561,7 +561,7 @@ public class InstantiatedIndexWriter implements Closeable {
// untokenized
String fieldVal = field.stringValue();
Token token = new Token(0, fieldVal.length(), "untokenized");
token.setTermBuffer(fieldVal);
token.setEmpty().append(fieldVal);
tokens.add(token);
fieldSetting.fieldLength++;
}
@ -596,10 +596,10 @@ public class InstantiatedIndexWriter implements Closeable {
for (Token token : eField_Tokens.getValue()) {
TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.term());
TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.toString());
if (termDocumentInformationFactory == null) {
termDocumentInformationFactory = new TermDocumentInformationFactory();
termDocumentInformationFactoryByTermText.put(token.term(), termDocumentInformationFactory);
termDocumentInformationFactoryByTermText.put(token.toString(), termDocumentInformationFactory);
}
//termDocumentInformationFactory.termFrequency++;

12
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
View File

@ -25,7 +25,7 @@ import java.util.List;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@ -278,7 +278,7 @@ public class TestIndicesEquals extends LuceneTestCase {
tokens.add(t);
tokens.add(createToken("fin", 7, 9));
TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class);
final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
Iterator<Token> it = tokens.iterator();
@Override
@ -601,16 +601,12 @@ public class TestIndicesEquals extends LuceneTestCase {
private static Token createToken(String term, int start, int offset)
{
Token token = new Token(start, offset);
token.setTermBuffer(term);
return token;
return new Token(term, start, offset);
}
private static Token createToken(String term, int start, int offset, String type)
{
Token token = new Token(start, offset, type);
token.setTermBuffer(term);
return token;
return new Token(term, start, offset, type);
}

6
lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
View File

@ -36,8 +36,8 @@ import jline.ConsoleReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
@ -303,14 +303,14 @@ class LuceneMethods {
int position = 0;
// Tokenize field and add to postingTable
TokenStream stream = analyzer.tokenStream(fieldName, reader);
TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
try {
while (stream.incrementToken()) {
position += (posIncrAtt.getPositionIncrement() - 1);
position++;
String name = termAtt.term();
String name = termAtt.toString();
Integer Count = tokenMap.get(name);
if (Count == null) { // not in there yet
tokenMap.put(name, Integer.valueOf(1)); //first one

20
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -30,9 +30,10 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.IndexReader;
@ -51,6 +52,7 @@ import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.RAMDirectory; // for javadocs
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants; // for javadocs
/**
@ -276,8 +278,8 @@ public class MemoryIndex implements Serializable {
return new TokenStream() {
private Iterator<T> iter = keywords.iterator();
private int start = 0;
private TermAttribute termAtt = addAttribute(TermAttribute.class);
private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
public boolean incrementToken() {
@ -289,8 +291,8 @@ public class MemoryIndex implements Serializable {
String term = obj.toString();
clearAttributes();
termAtt.setTermBuffer(term);
offsetAtt.setOffset(start, start+termAtt.termLength());
termAtt.setEmpty().append(term);
offsetAtt.setOffset(start, start+termAtt.length());
start += term.length() + 1; // separate words by 1 (blank) character
return true;
}
@ -340,13 +342,15 @@ public class MemoryIndex implements Serializable {
int numOverlapTokens = 0;
int pos = -1;
TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
TermToBytesRefAttribute termAtt = stream.addAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
BytesRef ref = new BytesRef(10);
stream.reset();
while (stream.incrementToken()) {
String term = termAtt.term();
termAtt.toBytesRef(ref);
// TODO: support non-UTF8 strings (like numerics) here
String term = ref.utf8ToString();
if (term.length() == 0) continue; // nothing to do
// if (DEBUG) System.err.println("token='" + term + "'");
numTokens++;

6
lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
View File

@ -26,7 +26,7 @@ import java.util.Iterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
@ -185,14 +185,14 @@ public class FuzzyLikeThisQuery extends Query
{
if(f.queryString==null) return;
TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
int corpusNumDocs=reader.numDocs();
Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects
HashSet<String> processedTerms=new HashSet<String>();
while (ts.incrementToken())
{
String term = termAtt.term();
String term = termAtt.toString();
if(!processedTerms.contains(term))
{
processedTerms.add(term);

6
lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
View File

@ -32,7 +32,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
@ -884,10 +884,10 @@ public final class MoreLikeThis {
TokenStream ts = analyzer.tokenStream(fieldName, r);
int tokenCount=0;
// for every token
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
String word = termAtt.term();
String word = termAtt.toString();
tokenCount++;
if(tokenCount>maxNumTokensParsed)
{

6
lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java
View File

@ -22,7 +22,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@ -86,12 +86,12 @@ public final class SimilarityQueries
throws IOException
{
TokenStream ts = a.tokenStream( field, new StringReader( body));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
BooleanQuery tmp = new BooleanQuery();
Set<String> already = new HashSet<String>(); // ignore dups
while (ts.incrementToken()) {
String word = termAtt.term();
String word = termAtt.toString();
// ignore opt stop words
if ( stop != null &&
stop.contains( word)) continue;

22
lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
View File

@ -24,7 +24,7 @@ import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Version;
@ -107,7 +107,7 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryParser.QueryPar
// get Analyzer from superclass and tokenize the term
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
TermAttribute termAtt = source.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
int countTokens = 0;
while (true) {
@ -116,7 +116,7 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryParser.QueryPar
} catch (IOException e) {
break;
}
String term = termAtt.term();
String term = termAtt.toString();
if (!"".equals(term)) {
try {
tlist.set(countTokens++, term);
@ -190,7 +190,7 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryParser.QueryPar
// get Analyzer from superclass and tokenize the term
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
List<String> tlist = new ArrayList<String>();
TermAttribute termAtt = source.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
while (true) {
try {
@ -198,7 +198,7 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryParser.QueryPar
} catch (IOException e) {
break;
}
tlist.add(termAtt.term());
tlist.add(termAtt.toString());
}
try {
@ -237,13 +237,13 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryParser.QueryPar
throws ParseException {
// get Analyzer from superclass and tokenize the term
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
TermAttribute termAtt = source.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
String nextToken = null;
boolean multipleTokens = false;
try {
if (source.incrementToken()) {
nextToken = termAtt.term();
nextToken = termAtt.toString();
}
multipleTokens = source.incrementToken();
} catch (IOException e) {
@ -273,13 +273,13 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryParser.QueryPar
throws ParseException {
// get Analyzer from superclass and tokenize the terms
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(part1));
TermAttribute termAtt = source.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
boolean multipleTokens = false;
// part1
try {
if (source.incrementToken()) {
part1 = termAtt.term();
part1 = termAtt.toString();
}
multipleTokens = source.incrementToken();
} catch (IOException e) {
@ -297,11 +297,11 @@ public class AnalyzingQueryParser extends org.apache.lucene.queryParser.QueryPar
// part2
source = getAnalyzer().tokenStream(field, new StringReader(part2));
termAtt = source.addAttribute(TermAttribute.class);
termAtt = source.addAttribute(CharTermAttribute.class);
try {
if (source.incrementToken()) {
part2 = termAtt.term();
part2 = termAtt.toString();
}
multipleTokens = source.incrementToken();
} catch (IOException e) {

10
lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
View File

@ -307,7 +307,7 @@ public class PrecedenceQueryParser implements PrecedenceQueryParserConstants {
List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
TermAttribute termAtt = source.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
try {
@ -328,7 +328,7 @@ public class PrecedenceQueryParser implements PrecedenceQueryParserConstants {
return null;
else if (list.size() == 1) {
source.restoreState(list.get(0));
return new TermQuery(new Term(field, termAtt.term()));
return new TermQuery(new Term(field, termAtt.toString()));
} else {
if (severalTokensAtSamePosition) {
if (positionCount == 1) {
@ -337,7 +337,7 @@ public class PrecedenceQueryParser implements PrecedenceQueryParserConstants {
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
TermQuery currentQuery = new TermQuery(
new Term(field, termAtt.term()));
new Term(field, termAtt.toString()));
q.add(currentQuery, BooleanClause.Occur.SHOULD);
}
return q;
@ -352,7 +352,7 @@ public class PrecedenceQueryParser implements PrecedenceQueryParserConstants {
mpq.add(multiTerms.toArray(new Term[0]));
multiTerms.clear();
}
multiTerms.add(new Term(field, termAtt.term()));
multiTerms.add(new Term(field, termAtt.toString()));
}
mpq.add(multiTerms.toArray(new Term[0]));
return mpq;
@ -363,7 +363,7 @@ public class PrecedenceQueryParser implements PrecedenceQueryParserConstants {
q.setSlop(phraseSlop);
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
q.add(new Term(field, termAtt.term()));
q.add(new Term(field, termAtt.toString()));
}
return q;
}

10
lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
View File

@ -331,7 +331,7 @@ public class PrecedenceQueryParser {
List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
TermAttribute termAtt = source.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
try {
@ -352,7 +352,7 @@ public class PrecedenceQueryParser {
return null;
else if (list.size() == 1) {
source.restoreState(list.get(0));
return new TermQuery(new Term(field, termAtt.term()));
return new TermQuery(new Term(field, termAtt.toString()));
} else {
if (severalTokensAtSamePosition) {
if (positionCount == 1) {
@ -361,7 +361,7 @@ public class PrecedenceQueryParser {
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
TermQuery currentQuery = new TermQuery(
new Term(field, termAtt.term()));
new Term(field, termAtt.toString()));
q.add(currentQuery, BooleanClause.Occur.SHOULD);
}
return q;
@ -376,7 +376,7 @@ public class PrecedenceQueryParser {
mpq.add(multiTerms.toArray(new Term[0]));
multiTerms.clear();
}
multiTerms.add(new Term(field, termAtt.term()));
multiTerms.add(new Term(field, termAtt.toString()));
}
mpq.add(multiTerms.toArray(new Term[0]));
return mpq;
@ -387,7 +387,7 @@ public class PrecedenceQueryParser {
q.setSlop(phraseSlop);
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
q.add(new Term(field, termAtt.term()));
q.add(new Term(field, termAtt.toString()));
}
return q;
}

14
lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
View File

@ -26,8 +26,8 @@ import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.queryParser.core.QueryNodeException;
import org.apache.lucene.queryParser.core.config.QueryConfigHandler;
import org.apache.lucene.queryParser.core.nodes.FieldQueryNode;
@ -162,11 +162,11 @@ public class AnalyzerQueryNodeProcessor extends QueryNodeProcessorImpl {
// ignore
}
if (!buffer.hasAttribute(TermAttribute.class)) {
if (!buffer.hasAttribute(CharTermAttribute.class)) {
return new NoTokenFoundQueryNode();
}
TermAttribute termAtt = buffer.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = buffer.getAttribute(CharTermAttribute.class);
if (numTokens == 0) {
return new NoTokenFoundQueryNode();
@ -177,7 +177,7 @@ public class AnalyzerQueryNodeProcessor extends QueryNodeProcessorImpl {
boolean hasNext;
hasNext = buffer.incrementToken();
assert hasNext == true;
term = termAtt.term();
term = termAtt.toString();
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
@ -197,7 +197,7 @@ public class AnalyzerQueryNodeProcessor extends QueryNodeProcessorImpl {
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
term = termAtt.term();
term = termAtt.toString();
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
@ -224,7 +224,7 @@ public class AnalyzerQueryNodeProcessor extends QueryNodeProcessorImpl {
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
term = termAtt.term();
term = termAtt.toString();
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
@ -290,7 +290,7 @@ public class AnalyzerQueryNodeProcessor extends QueryNodeProcessorImpl {
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
term = termAtt.term();
term = termAtt.toString();
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();

12
lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
View File

@ -23,8 +23,8 @@ import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FuzzyQuery;
@ -68,7 +68,7 @@ public class TestPrecedenceQueryParser extends LocalizedTestCase {
boolean inPhrase = false;
int savedStart = 0, savedEnd = 0;
TermAttribute termAtt = addAttribute(TermAttribute.class);
CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
@ -76,19 +76,19 @@ public class TestPrecedenceQueryParser extends LocalizedTestCase {
clearAttributes();
if (inPhrase) {
inPhrase = false;
termAtt.setTermBuffer("phrase2");
termAtt.setEmpty().append("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else
while(input.incrementToken())
if (termAtt.term().equals("phrase")) {
if (termAtt.toString().equals("phrase")) {
inPhrase = true;
savedStart = offsetAtt.startOffset();
savedEnd = offsetAtt.endOffset();
termAtt.setTermBuffer("phrase1");
termAtt.setEmpty().append("phrase1");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else if (!termAtt.term().equals("stop"))
} else if (!termAtt.toString().equals("stop"))
return true;
return false;
}

29
lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
View File

@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.queryParser.core.QueryNodeException;
import org.apache.lucene.queryParser.standard.config.DefaultOperatorAttribute.Operator;
@ -163,24 +163,19 @@ public class TestMultiAnalyzerQPHelper extends LuceneTestCase {
private int prevStartOffset;
private int prevEndOffset;
TermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
OffsetAttribute offsetAtt;
TypeAttribute typeAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
public TestFilter(TokenStream in) {
super(in);
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}
@Override
public final boolean incrementToken() throws java.io.IOException {
if (multiToken > 0) {
termAtt.setTermBuffer("multi" + (multiToken + 1));
termAtt.setEmpty().append("multi" + (multiToken + 1));
offsetAtt.setOffset(prevStartOffset, prevEndOffset);
typeAtt.setType(prevType);
posIncrAtt.setPositionIncrement(0);
@ -194,7 +189,7 @@ public class TestMultiAnalyzerQPHelper extends LuceneTestCase {
prevType = typeAtt.type();
prevStartOffset = offsetAtt.startOffset();
prevEndOffset = offsetAtt.endOffset();
String text = termAtt.term();
String text = termAtt.toString();
if (text.equals("triplemulti")) {
multiToken = 2;
return true;
@ -228,21 +223,19 @@ public class TestMultiAnalyzerQPHelper extends LuceneTestCase {
private class TestPosIncrementFilter extends TokenFilter {
TermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
public TestPosIncrementFilter(TokenStream in) {
super(in);
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
@Override
public final boolean incrementToken() throws java.io.IOException {
while (input.incrementToken()) {
if (termAtt.term().equals("the")) {
if (termAtt.toString().equals("the")) {
// stopword, do nothing
} else if (termAtt.term().equals("quick")) {
} else if (termAtt.toString().equals("quick")) {
posIncrAtt.setPositionIncrement(2);
return true;
} else {

29
lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
View File

@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.util.LuceneTestCase;
@ -157,24 +157,19 @@ public class TestMultiAnalyzerWrapper extends LuceneTestCase {
private int prevStartOffset;
private int prevEndOffset;
TermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
OffsetAttribute offsetAtt;
TypeAttribute typeAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
public TestFilter(TokenStream in) {
super(in);
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}
@Override
public final boolean incrementToken() throws java.io.IOException {
if (multiToken > 0) {
termAtt.setTermBuffer("multi" + (multiToken + 1));
termAtt.setEmpty().append("multi" + (multiToken + 1));
offsetAtt.setOffset(prevStartOffset, prevEndOffset);
typeAtt.setType(prevType);
posIncrAtt.setPositionIncrement(0);
@ -188,7 +183,7 @@ public class TestMultiAnalyzerWrapper extends LuceneTestCase {
prevType = typeAtt.type();
prevStartOffset = offsetAtt.startOffset();
prevEndOffset = offsetAtt.endOffset();
String text = termAtt.term();
String text = termAtt.toString();
if (text.equals("triplemulti")) {
multiToken = 2;
return true;
@ -222,21 +217,19 @@ public class TestMultiAnalyzerWrapper extends LuceneTestCase {
private class TestPosIncrementFilter extends TokenFilter {
TermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
public TestPosIncrementFilter(TokenStream in) {
super(in);
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
@Override
public final boolean incrementToken() throws java.io.IOException {
while (input.incrementToken()) {
if (termAtt.term().equals("the")) {
if (termAtt.toString().equals("the")) {
// stopword, do nothing
} else if (termAtt.term().equals("quick")) {
} else if (termAtt.toString().equals("quick")) {
posIncrAtt.setPositionIncrement(2);
return true;
} else {

26
lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
View File

@ -37,8 +37,8 @@ import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
@ -96,8 +96,8 @@ public class TestQPHelper extends LocalizedTestCase {
public static Analyzer qpAnalyzer = new QPTestAnalyzer();
public static final class QPTestFilter extends TokenFilter {
TermAttribute termAtt;
OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Filter which discards the token 'stop' and which expands the token
@ -105,8 +105,6 @@ public class TestQPHelper extends LocalizedTestCase {
*/
public QPTestFilter(TokenStream in) {
super(in);
termAtt = addAttribute(TermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
boolean inPhrase = false;
@ -117,19 +115,19 @@ public class TestQPHelper extends LocalizedTestCase {
if (inPhrase) {
inPhrase = false;
clearAttributes();
termAtt.setTermBuffer("phrase2");
termAtt.setEmpty().append("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else
while (input.incrementToken()) {
if (termAtt.term().equals("phrase")) {
if (termAtt.toString().equals("phrase")) {
inPhrase = true;
savedStart = offsetAtt.startOffset();
savedEnd = offsetAtt.endOffset();
termAtt.setTermBuffer("phrase1");
termAtt.setEmpty().append("phrase1");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else if (!termAtt.term().equals("stop"))
} else if (!termAtt.toString().equals("stop"))
return true;
}
return false;
@ -1158,7 +1156,7 @@ public class TestQPHelper extends LocalizedTestCase {
private class CannedTokenStream extends TokenStream {
private int upto = 0;
final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
final TermAttribute term = addAttribute(TermAttribute.class);
final CharTermAttribute term = addAttribute(CharTermAttribute.class);
@Override
public boolean incrementToken() {
clearAttributes();
@ -1167,16 +1165,16 @@ public class TestQPHelper extends LocalizedTestCase {
}
if (upto == 0) {
posIncr.setPositionIncrement(1);
term.setTermBuffer("a");
term.setEmpty().append("a");
} else if (upto == 1) {
posIncr.setPositionIncrement(1);
term.setTermBuffer("b");
term.setEmpty().append("b");
} else if (upto == 2) {
posIncr.setPositionIncrement(0);
term.setTermBuffer("c");
term.setEmpty().append("c");
} else {
posIncr.setPositionIncrement(0);
term.setTermBuffer("d");
term.setEmpty().append("d");
}
upto++;
return true;

16
lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
View File

@ -36,7 +36,7 @@ import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
@ -93,8 +93,8 @@ public class TestQueryParserWrapper extends LocalizedTestCase {
public static Analyzer qpAnalyzer = new QPTestAnalyzer();
public static final class QPTestFilter extends TokenFilter {
TermAttribute termAtt;
OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Filter which discards the token 'stop' and which expands the token
@ -102,8 +102,6 @@ public class TestQueryParserWrapper extends LocalizedTestCase {
*/
public QPTestFilter(TokenStream in) {
super(in);
termAtt = addAttribute(TermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
boolean inPhrase = false;
@ -114,19 +112,19 @@ public class TestQueryParserWrapper extends LocalizedTestCase {
if (inPhrase) {
inPhrase = false;
clearAttributes();
termAtt.setTermBuffer("phrase2");
termAtt.setEmpty().append("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else
while (input.incrementToken()) {
if (termAtt.term().equals("phrase")) {
if (termAtt.toString().equals("phrase")) {
inPhrase = true;
savedStart = offsetAtt.startOffset();
savedEnd = offsetAtt.endOffset();
termAtt.setTermBuffer("phrase1");
termAtt.setEmpty().append("phrase1");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else if (!termAtt.term().equals("stop"))
} else if (!termAtt.toString().equals("stop"))
return true;
}
return false;

6
lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
View File

@ -29,7 +29,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
@ -117,10 +117,10 @@ public final class SynExpand {
// [1] Parse query into separate words so that when we expand we can avoid dups
TokenStream ts = a.tokenStream( field, new StringReader( query));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
String word = termAtt.term();
String word = termAtt.toString();
if ( already.add( word))
top.add( word);
}

6
lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
View File

@ -28,7 +28,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
@ -125,10 +125,10 @@ public class SynLookup {
// [1] Parse query into separate words so that when we expand we can avoid dups
TokenStream ts = a.tokenStream( field, new StringReader( query));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
String word = termAtt.term();
String word = termAtt.toString();
if ( already.add( word))
top.add( word);
}

16
lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java
View File

@ -21,8 +21,8 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;
@ -45,9 +45,9 @@ public class SynonymTokenFilter extends TokenFilter {
private AttributeSource.State current = null;
private int todo = 0;
private TermAttribute termAtt;
private TypeAttribute typeAtt;
private PositionIncrementAttribute posIncrAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
/**
* Creates an instance for the given underlying stream and synonym table.
@ -71,10 +71,6 @@ public class SynonymTokenFilter extends TokenFilter {
this.synonyms = synonyms;
this.maxSynonyms = maxSynonyms;
this.termAtt = addAttribute(TermAttribute.class);
this.typeAtt = addAttribute(TypeAttribute.class);
this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
/** Returns the next token in the stream, or null at EOS. */
@ -89,7 +85,7 @@ public class SynonymTokenFilter extends TokenFilter {
if (!input.incrementToken()) return false; // EOS; iterator exhausted
stack = synonyms.getSynonyms(termAtt.term()); // push onto stack
stack = synonyms.getSynonyms(termAtt.toString()); // push onto stack
if (stack.length > maxSynonyms) randomize(stack);
index = 0;
current = captureState();
@ -110,7 +106,7 @@ public class SynonymTokenFilter extends TokenFilter {
*/
protected boolean createToken(String synonym, AttributeSource.State current) {
restoreState(current);
termAtt.setTermBuffer(synonym);
termAtt.setEmpty().append(synonym);
typeAtt.setType(SYNONYM_TOKEN_TYPE);
posIncrAtt.setPositionIncrement(0);
return true;

6
lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
View File

@ -10,7 +10,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.search.similar.MoreLikeThisQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.xmlparser.DOMUtils;
@ -77,11 +77,11 @@ public class LikeThisQueryBuilder implements QueryBuilder {
for (int i = 0; i < fields.length; i++)
{
TokenStream ts = analyzer.tokenStream(fields[i],new StringReader(stopWords));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
try
{
while(ts.incrementToken()) {
stopWordsSet.add(termAtt.term());
stopWordsSet.add(termAtt.toString());
}
}
catch(IOException ioe)

6
lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
View File

@ -6,7 +6,7 @@ import java.util.ArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
@ -56,10 +56,10 @@ public class SpanOrTermsBuilder extends SpanBuilderBase
{
ArrayList<SpanQuery> clausesList=new ArrayList<SpanQuery>();
TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, termAtt.term()));
SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, termAtt.toString()));
clausesList.add(stq);
}
SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));

8
lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
View File

@ -5,7 +5,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermsFilter;
@ -57,7 +57,7 @@ public class TermsFilterBuilder implements FilterBuilder
String text = DOMUtils.getNonBlankTextOrFail(e);
String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
try
{
@ -65,11 +65,11 @@ public class TermsFilterBuilder implements FilterBuilder
while (ts.incrementToken()) {
if (term == null)
{
term = new Term(fieldName, termAtt.term());
term = new Term(fieldName, termAtt.toString());
} else
{
// create from previous to save fieldName.intern overhead
term = term.createTerm(termAtt.term());
term = term.createTerm(termAtt.toString());
}
tf.addTerm(term);
}

8
lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
View File

@ -5,7 +5,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@ -57,16 +57,16 @@ public class TermsQueryBuilder implements QueryBuilder {
TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
try
{
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
Term term = null;
while (ts.incrementToken()) {
if (term == null)
{
term = new Term(fieldName, termAtt.term());
term = new Term(fieldName, termAtt.toString());
} else
{
// create from previous to save fieldName.intern overhead
term = term.createTerm(termAtt.term());
term = term.createTerm(termAtt.toString());
}
bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
}

13
modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java
View File

@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link ArabicNormalizer} to normalize the orthography.
@ -29,21 +29,18 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
*/
public final class ArabicNormalizationFilter extends TokenFilter {
private final ArabicNormalizer normalizer;
private final TermAttribute termAtt;
private final ArabicNormalizer normalizer = new ArabicNormalizer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public ArabicNormalizationFilter(TokenStream input) {
super(input);
normalizer = new ArabicNormalizer();
termAtt = addAttribute(TermAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
int newlen = normalizer.normalize(termAtt.termBuffer(), termAtt.termLength());
termAtt.setTermLength(newlen);
int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length());
termAtt.setLength(newlen);
return true;
}
return false;

16
modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; // javadoc
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link ArabicStemmer} to stem Arabic words..
@ -35,24 +35,20 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
* @see KeywordMarkerFilter */
public final class ArabicStemFilter extends TokenFilter {
private final ArabicStemmer stemmer;
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final ArabicStemmer stemmer = new ArabicStemmer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
public ArabicStemFilter(TokenStream input) {
super(input);
stemmer = new ArabicStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if(!keywordAttr.isKeyword()) {
final int newlen = stemmer.stem(termAtt.termBuffer(), termAtt.termLength());
termAtt.setTermLength(newlen);
final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
termAtt.setLength(newlen);
}
return true;
} else {

15
modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; // for java
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link BulgarianStemmer} to stem Bulgarian
@ -35,23 +35,20 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
* </p>
*/
public final class BulgarianStemFilter extends TokenFilter {
private final BulgarianStemmer stemmer;
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final BulgarianStemmer stemmer = new BulgarianStemmer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
public BulgarianStemFilter(final TokenStream input) {
super(input);
stemmer = new BulgarianStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if(!keywordAttr.isKeyword()) {
final int newlen = stemmer.stem(termAtt.termBuffer(), termAtt.termLength());
termAtt.setTermLength(newlen);
final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
termAtt.setLength(newlen);
}
return true;
} else {

15
modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; // for java
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link BrazilianStemmer}.
@ -41,10 +41,10 @@ public final class BrazilianStemFilter extends TokenFilter {
/**
* {@link BrazilianStemmer} in use by this filter.
*/
private BrazilianStemmer stemmer = null;
private BrazilianStemmer stemmer = new BrazilianStemmer();
private Set<?> exclusions = null;
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
/**
* Creates a new BrazilianStemFilter
@ -53,9 +53,6 @@ public final class BrazilianStemFilter extends TokenFilter {
*/
public BrazilianStemFilter(TokenStream in) {
super(in);
stemmer = new BrazilianStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
}
/**
@ -74,13 +71,13 @@ public final class BrazilianStemFilter extends TokenFilter {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
final String term = termAtt.term();
final String term = termAtt.toString();
// Check the exclusion table.
if (!keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains(term))) {
final String s = stemmer.stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.equals(term))
termAtt.setTermBuffer(s);
termAtt.setEmpty().append(s);
}
return true;
} else {

19
modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
View File

@ -22,7 +22,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;
@ -98,9 +98,9 @@ public final class CJKTokenizer extends Tokenizer {
*/
private boolean preIsTokened = false;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private TypeAttribute typeAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
//~ Constructors -----------------------------------------------------------
@ -111,23 +111,14 @@ public final class CJKTokenizer extends Tokenizer {
*/
public CJKTokenizer(Reader in) {
super(in);
init();
}
public CJKTokenizer(AttributeSource source, Reader in) {
super(source, in);
init();
}
public CJKTokenizer(AttributeFactory factory, Reader in) {
super(factory, in);
init();
}
private void init() {
termAtt = addAttribute(TermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}
//~ Methods ----------------------------------------------------------------
@ -287,7 +278,7 @@ public final class CJKTokenizer extends Tokenizer {
}
if (length > 0) {
termAtt.setTermBuffer(buffer, 0, length);
termAtt.copyBuffer(buffer, 0, length);
offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
typeAtt.setType(TOKEN_TYPE_NAMES[tokenType]);
return true;

9
modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java
View File

@ -23,7 +23,7 @@ import java.util.Arrays;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
@ -61,21 +61,20 @@ public final class ChineseFilter extends TokenFilter {
private CharArraySet stopTable;
private TermAttribute termAtt;
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public ChineseFilter(TokenStream in) {
super(in);
stopTable = new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(STOP_WORDS), false);
termAtt = addAttribute(TermAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
char text[] = termAtt.termBuffer();
int termLength = termAtt.termLength();
char text[] = termAtt.buffer();
int termLength = termAtt.length();
// why not key off token type here assuming ChineseTokenizer comes first?
if (!stopTable.contains(text, 0, termLength)) {

18
modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
View File

@ -23,8 +23,8 @@ import java.io.Reader;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.AttributeSource;
@ -62,24 +62,16 @@ public final class ChineseTokenizer extends Tokenizer {
public ChineseTokenizer(Reader in) {
super(in);
init();
}
public ChineseTokenizer(AttributeSource source, Reader in) {
super(source, in);
init();
}
public ChineseTokenizer(AttributeFactory factory, Reader in) {
super(factory, in);
init();
}
private void init() {
termAtt = addAttribute(TermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
private int offset = 0, bufferIndex=0, dataLen=0;
private final static int MAX_WORD_LEN = 255;
private final static int IO_BUFFER_SIZE = 1024;
@ -90,8 +82,8 @@ public final class ChineseTokenizer extends Tokenizer {
private int length;
private int start;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final void push(char c) {
@ -105,7 +97,7 @@ public final class ChineseTokenizer extends Tokenizer {
if (length>0) {
//System.out.println(new String(buffer, 0,
//length));
termAtt.setTermBuffer(buffer, 0, length);
termAtt.copyBuffer(buffer, 0, length);
offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
return true;
}

29
modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
View File

@ -30,7 +30,7 @@ import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
@ -69,12 +69,12 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
protected final int maxSubwordSize;
protected final boolean onlyLongestMatch;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private FlagsAttribute flagsAtt;
private PositionIncrementAttribute posIncAtt;
private TypeAttribute typeAtt;
private PayloadAttribute payloadAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
private final Token wrapper = new Token();
/**
@ -160,13 +160,6 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
this.dictionary = new CharArraySet(matchVersion, dictionary.size(), false);
addAllLowerCase(this.dictionary, dictionary);
}
termAtt = addAttribute(TermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
flagsAtt = addAttribute(FlagsAttribute.class);
posIncAtt = addAttribute(PositionIncrementAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
}
/**
@ -192,7 +185,7 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
private final void setToken(final Token token) throws IOException {
clearAttributes();
termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
termAtt.copyBuffer(token.buffer(), 0, token.length());
flagsAtt.setFlags(token.getFlags());
typeAtt.setType(token.type());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
@ -210,7 +203,7 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
if (!input.incrementToken())
return false;
wrapper.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
wrapper.copyBuffer(termAtt.buffer(), 0, termAtt.length());
wrapper.setStartOffset(offsetAtt.startOffset());
wrapper.setEndOffset(offsetAtt.endOffset());
wrapper.setFlags(flagsAtt.getFlags());
@ -248,7 +241,7 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
protected final Token createToken(final int offset, final int length,
final Token prototype) {
int newStart = prototype.startOffset() + offset;
Token t = prototype.clone(prototype.termBuffer(), offset, length, newStart, newStart+length);
Token t = prototype.clone(prototype.buffer(), offset, length, newStart, newStart+length);
t.setPositionIncrement(0);
return t;
}
@ -258,7 +251,7 @@ public abstract class CompoundWordTokenFilterBase extends TokenFilter {
tokens.add((Token) token.clone());
// Only words longer than minWordSize get processed
if (token.termLength() < this.minWordSize) {
if (token.length() < this.minWordSize) {
return;
}

10
modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
View File

@ -191,22 +191,22 @@ public class DictionaryCompoundWordTokenFilter extends CompoundWordTokenFilterBa
@Override
protected void decomposeInternal(final Token token) {
// Only words longer than minWordSize get processed
if (token.termLength() < this.minWordSize) {
if (token.length() < this.minWordSize) {
return;
}
char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.termBuffer());
char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.buffer());
for (int i=0;i<token.termLength()-this.minSubwordSize;++i) {
for (int i=0;i<token.length()-this.minSubwordSize;++i) {
Token longestMatchToken=null;
for (int j=this.minSubwordSize-1;j<this.maxSubwordSize;++j) {
if(i+j>token.termLength()) {
if(i+j>token.length()) {
break;
}
if(dictionary.contains(lowerCaseTermBuffer, i, j)) {
if (this.onlyLongestMatch) {
if (longestMatchToken!=null) {
if (longestMatchToken.termLength()<j) {
if (longestMatchToken.length()<j) {
longestMatchToken=createToken(i,j,token);
}
} else {

10
modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
View File

@ -302,15 +302,15 @@ public class HyphenationCompoundWordTokenFilter extends
@Override
protected void decomposeInternal(final Token token) {
// get the hyphenation points
Hyphenation hyphens = hyphenator.hyphenate(token.termBuffer(), 0, token
.termLength(), 1, 1);
Hyphenation hyphens = hyphenator.hyphenate(token.buffer(), 0, token
.length(), 1, 1);
// No hyphen points found -> exit
if (hyphens == null) {
return;
}
final int[] hyp = hyphens.getHyphenationPoints();
char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.termBuffer());
char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.buffer());
for (int i = 0; i < hyp.length; ++i) {
int remaining = hyp.length - i;
@ -335,7 +335,7 @@ public class HyphenationCompoundWordTokenFilter extends
if (dictionary == null || dictionary.contains(lowerCaseTermBuffer, start, partLength)) {
if (this.onlyLongestMatch) {
if (longestMatchToken != null) {
if (longestMatchToken.termLength() < partLength) {
if (longestMatchToken.length() < partLength) {
longestMatchToken = createToken(start, partLength, token);
}
} else {
@ -352,7 +352,7 @@ public class HyphenationCompoundWordTokenFilter extends
// characters
if (this.onlyLongestMatch) {
if (longestMatchToken != null) {
if (longestMatchToken.termLength() < partLength - 1) {
if (longestMatchToken.length() < partLength - 1) {
longestMatchToken = createToken(start, partLength - 1, token);
}
} else {

15
modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java
View File

@ -6,7 +6,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; // for java
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@ -37,23 +37,20 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
* @see KeywordMarkerFilter
*/
public final class CzechStemFilter extends TokenFilter {
private final CzechStemmer stemmer;
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final CzechStemmer stemmer = new CzechStemmer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
public CzechStemFilter(TokenStream input) {
super(input);
stemmer = new CzechStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if(!keywordAttr.isKeyword()) {
final int newlen = stemmer.stem(termAtt.termBuffer(), termAtt.termLength());
termAtt.setTermLength(newlen);
final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
termAtt.setLength(newlen);
}
return true;
} else {

15
modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; // for java
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that stems German words.
@ -45,11 +45,11 @@ public final class GermanStemFilter extends TokenFilter
/**
* The actual token in the input stream.
*/
private GermanStemmer stemmer = null;
private GermanStemmer stemmer = new GermanStemmer();
private Set<?> exclusionSet = null;
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
/**
* Creates a {@link GermanStemFilter} instance
@ -58,9 +58,6 @@ public final class GermanStemFilter extends TokenFilter
public GermanStemFilter( TokenStream in )
{
super(in);
stemmer = new GermanStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
}
/**
@ -80,13 +77,13 @@ public final class GermanStemFilter extends TokenFilter
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
String term = termAtt.term();
String term = termAtt.toString();
// Check the exclusion table.
if (!keywordAttr.isKeyword() && (exclusionSet == null || !exclusionSet.contains(term))) {
String s = stemmer.stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.equals(term))
termAtt.setTermBuffer(s);
termAtt.setEmpty().append(s);
}
return true;
} else {

15
modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java
View File

@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link PersianNormalizer} to normalize the
@ -30,22 +30,19 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
*/
public final class PersianNormalizationFilter extends TokenFilter {
private final PersianNormalizer normalizer;
private final TermAttribute termAtt;
private final PersianNormalizer normalizer = new PersianNormalizer();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public PersianNormalizationFilter(TokenStream input) {
super(input);
normalizer = new PersianNormalizer();
termAtt = addAttribute(TermAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
final int newlen = normalizer.normalize(termAtt.termBuffer(), termAtt
.termLength());
termAtt.setTermLength(newlen);
final int newlen = normalizer.normalize(termAtt.buffer(),
termAtt.length());
termAtt.setLength(newlen);
return true;
}
return false;

15
modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
View File

@ -23,7 +23,7 @@ import java.util.Arrays;
import org.apache.lucene.analysis.standard.StandardTokenizer; // for javadocs
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
@ -37,7 +37,7 @@ import org.apache.lucene.util.Version;
*/
public final class ElisionFilter extends TokenFilter {
private CharArraySet articles = CharArraySet.EMPTY_SET;
private final TermAttribute termAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(
"l", "m", "t", "qu", "n", "s", "j"), true));
@ -100,7 +100,6 @@ public final class ElisionFilter extends TokenFilter {
super(input);
this.articles = CharArraySet.unmodifiableSet(
new CharArraySet(matchVersion, articles, true));
termAtt = addAttribute(TermAttribute.class);
}
/**
@ -115,13 +114,13 @@ public final class ElisionFilter extends TokenFilter {
}
/**
* Increments the {@link TokenStream} with a {@link TermAttribute} without elisioned start
* Increments the {@link TokenStream} with a {@link CharTermAttribute} without elisioned start
*/
@Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
char[] termBuffer = termAtt.termBuffer();
int termLength = termAtt.termLength();
char[] termBuffer = termAtt.buffer();
int termLength = termAtt.length();
int minPoz = Integer.MAX_VALUE;
for (int i = 0; i < apostrophes.length; i++) {
@ -137,8 +136,8 @@ public final class ElisionFilter extends TokenFilter {
// An apostrophe has been found. If the prefix is an article strip it off.
if (minPoz != Integer.MAX_VALUE
&& articles.contains(termAtt.termBuffer(), 0, minPoz)) {
termAtt.setTermBuffer(termAtt.termBuffer(), minPoz + 1, termAtt.termLength() - (minPoz + 1));
&& articles.contains(termAtt.buffer(), 0, minPoz)) {
termAtt.copyBuffer(termAtt.buffer(), minPoz + 1, termAtt.length() - (minPoz + 1));
}
return true;

17
modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
import java.util.HashSet;
@ -51,17 +51,14 @@ public final class FrenchStemFilter extends TokenFilter {
/**
* The actual token in the input stream.
*/
private FrenchStemmer stemmer = null;
private FrenchStemmer stemmer = new FrenchStemmer();
private Set<?> exclusions = null;
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
public FrenchStemFilter( TokenStream in ) {
super(in);
stemmer = new FrenchStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
super(in);
}
/**
@ -82,14 +79,14 @@ public final class FrenchStemFilter extends TokenFilter {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
String term = termAtt.term();
String term = termAtt.toString();
// Check the exclusion table
if ( !keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains( term )) ) {
String s = stemmer.stem( term );
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.equals( term ) )
termAtt.setTermBuffer(s);
termAtt.setEmpty().append(s);
}
return true;
} else {

8
modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; // javadoc
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link HindiNormalizer} to normalize the
@ -39,7 +39,7 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
public final class HindiNormalizationFilter extends TokenFilter {
private final HindiNormalizer normalizer = new HindiNormalizer();
private final TermAttribute termAtt = addAttribute(TermAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
public HindiNormalizationFilter(TokenStream input) {
@ -50,8 +50,8 @@ public final class HindiNormalizationFilter extends TokenFilter {
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAtt.isKeyword())
termAtt.setTermLength(normalizer.normalize(termAtt.termBuffer(),
termAtt.termLength()));
termAtt.setLength(normalizer.normalize(termAtt.buffer(),
termAtt.length()));
return true;
}
return false;

6
modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java
View File

@ -22,13 +22,13 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link HindiStemmer} to stem Hindi words.
*/
public final class HindiStemFilter extends TokenFilter {
private final TermAttribute termAtt = addAttribute(TermAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
private final HindiStemmer stemmer = new HindiStemmer();
@ -40,7 +40,7 @@ public final class HindiStemFilter extends TokenFilter {
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAtt.isKeyword())
termAtt.setTermLength(stemmer.stem(termAtt.termBuffer(), termAtt.termLength()));
termAtt.setLength(stemmer.stem(termAtt.buffer(), termAtt.length()));
return true;
} else {
return false;

6
modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java
View File

@ -21,14 +21,14 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that applies {@link IndicNormalizer} to normalize text
* in Indian Languages.
*/
public final class IndicNormalizationFilter extends TokenFilter {
private final TermAttribute termAtt = addAttribute(TermAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final IndicNormalizer normalizer = new IndicNormalizer();
public IndicNormalizationFilter(TokenStream input) {
@ -38,7 +38,7 @@ public final class IndicNormalizationFilter extends TokenFilter {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
termAtt.setTermLength(normalizer.normalize(termAtt.termBuffer(), termAtt.termLength()));
termAtt.setLength(normalizer.normalize(termAtt.buffer(), termAtt.length()));
return true;
} else {
return false;

14
modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
View File

@ -30,8 +30,8 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
@ -332,8 +332,8 @@ public final class PatternAnalyzer extends Analyzer {
private Matcher matcher;
private int pos = 0;
private static final Locale locale = Locale.getDefault();
private TermAttribute termAtt = addAttribute(TermAttribute.class);
private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
public PatternTokenizer(String str, Pattern pattern, boolean toLowerCase) {
this.str = str;
@ -360,7 +360,7 @@ public final class PatternAnalyzer extends Analyzer {
if (start != end) { // non-empty match (header/trailer)
String text = str.substring(start, end);
if (toLowerCase) text = text.toLowerCase(locale);
termAtt.setTermBuffer(text);
termAtt.setEmpty().append(text);
offsetAtt.setOffset(start, end);
return true;
}
@ -392,8 +392,8 @@ public final class PatternAnalyzer extends Analyzer {
private final boolean toLowerCase;
private final Set<?> stopWords;
private static final Locale locale = Locale.getDefault();
private TermAttribute termAtt = addAttribute(TermAttribute.class);
private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
public FastStringTokenizer(String str, boolean isLetter, boolean toLowerCase, Set<?> stopWords) {
this.str = str;
@ -446,7 +446,7 @@ public final class PatternAnalyzer extends Analyzer {
{
return false;
}
termAtt.setTermBuffer(text);
termAtt.setEmpty().append(text);
offsetAtt.setOffset(start, i);
return true;
}

16
modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.index.Payload;
@ -44,14 +44,14 @@ public class PrefixAwareTokenFilter extends TokenStream {
private TokenStream prefix;
private TokenStream suffix;
private TermAttribute termAtt;
private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncrAtt;
private PayloadAttribute payloadAtt;
private OffsetAttribute offsetAtt;
private TypeAttribute typeAtt;
private FlagsAttribute flagsAtt;
private TermAttribute p_termAtt;
private CharTermAttribute p_termAtt;
private PositionIncrementAttribute p_posIncrAtt;
private PayloadAttribute p_payloadAtt;
private OffsetAttribute p_offsetAtt;
@ -64,14 +64,14 @@ public class PrefixAwareTokenFilter extends TokenStream {
this.prefix = prefix;
prefixExhausted = false;
termAtt = addAttribute(TermAttribute.class);
termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
flagsAtt = addAttribute(FlagsAttribute.class);
p_termAtt = prefix.addAttribute(TermAttribute.class);
p_termAtt = prefix.addAttribute(CharTermAttribute.class);
p_posIncrAtt = prefix.addAttribute(PositionIncrementAttribute.class);
p_payloadAtt = prefix.addAttribute(PayloadAttribute.class);
p_offsetAtt = prefix.addAttribute(OffsetAttribute.class);
@ -115,7 +115,7 @@ public class PrefixAwareTokenFilter extends TokenStream {
private void setCurrentToken(Token token) {
if (token == null) return;
clearAttributes();
termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
termAtt.copyBuffer(token.buffer(), 0, token.length());
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
flagsAtt.setFlags(token.getFlags());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
@ -125,7 +125,7 @@ public class PrefixAwareTokenFilter extends TokenStream {
private Token getNextPrefixInputToken(Token token) throws IOException {
if (!prefix.incrementToken()) return null;
token.setTermBuffer(p_termAtt.termBuffer(), 0, p_termAtt.termLength());
token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length());
token.setPositionIncrement(p_posIncrAtt.getPositionIncrement());
token.setFlags(p_flagsAtt.getFlags());
token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset());
@ -136,7 +136,7 @@ public class PrefixAwareTokenFilter extends TokenStream {
private Token getNextSuffixInputToken(Token token) throws IOException {
if (!suffix.incrementToken()) return null;
token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
token.setPositionIncrement(posIncrAtt.getPositionIncrement());
token.setFlags(flagsAtt.getFlags());
token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());

4
modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
View File

@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenStream} containing a single token.
@ -41,7 +41,7 @@ public final class SingleTokenTokenStream extends TokenStream {
assert token != null;
this.singleToken = (Token) token.clone();
tokenAtt = (AttributeImpl) addAttribute(TermAttribute.class);
tokenAtt = (AttributeImpl) addAttribute(CharTermAttribute.class);
assert (tokenAtt instanceof Token);
}

8
modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
View File

@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.util.Version;
@ -34,7 +34,7 @@ import org.apache.lucene.util.Version;
public final class StemmerOverrideFilter extends TokenFilter {
private final CharArrayMap<String> dictionary;
private final TermAttribute termAtt = addAttribute(TermAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
/**
@ -56,9 +56,9 @@ public final class StemmerOverrideFilter extends TokenFilter {
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms
String stem = dictionary.get(termAtt.termBuffer(), 0, termAtt.termLength());
String stem = dictionary.get(termAtt.buffer(), 0, termAtt.length());
if (stem != null) {
termAtt.setTermBuffer(stem);
termAtt.setEmpty().append(stem);
keywordAtt.setKeyword(true);
}
}

14
modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
View File

@ -20,7 +20,7 @@ package org.apache.lucene.analysis.ngram;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
@ -72,8 +72,8 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
private int curGramSize;
private int tokStart;
private final TermAttribute termAtt;
private final OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Creates EdgeNGramTokenFilter that can generate n-grams in the sizes of the given range
@ -101,8 +101,6 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
this.minGram = minGram;
this.maxGram = maxGram;
this.side = side;
this.termAtt = addAttribute(TermAttribute.class);
this.offsetAtt = addAttribute(OffsetAttribute.class);
}
/**
@ -124,8 +122,8 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
if (!input.incrementToken()) {
return false;
} else {
curTermBuffer = termAtt.termBuffer().clone();
curTermLength = termAtt.termLength();
curTermBuffer = termAtt.buffer().clone();
curTermLength = termAtt.length();
curGramSize = minGram;
tokStart = offsetAtt.startOffset();
}
@ -138,7 +136,7 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
int end = start + curGramSize;
clearAttributes();
offsetAtt.setOffset(tokStart + start, tokStart + end);
termAtt.setTermBuffer(curTermBuffer, start, curGramSize);
termAtt.copyBuffer(curTermBuffer, start, curGramSize);
curGramSize++;
return true;
}

12
modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
View File

@ -18,8 +18,8 @@ package org.apache.lucene.analysis.ngram;
*/
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.AttributeSource;
import java.io.IOException;
@ -37,8 +37,8 @@ public final class EdgeNGramTokenizer extends Tokenizer {
public static final int DEFAULT_MAX_GRAM_SIZE = 1;
public static final int DEFAULT_MIN_GRAM_SIZE = 1;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/** Specifies which side of the input the n-gram should be generated from */
public static enum Side {
@ -173,10 +173,6 @@ public final class EdgeNGramTokenizer extends Tokenizer {
this.minGram = minGram;
this.maxGram = maxGram;
this.side = side;
this.termAtt = addAttribute(TermAttribute.class);
this.offsetAtt = addAttribute(OffsetAttribute.class);
}
/** Returns the next token in the stream, or null at EOS. */
@ -206,7 +202,7 @@ public final class EdgeNGramTokenizer extends Tokenizer {
// grab gramSize chars from front or back
int start = side == Side.FRONT ? 0 : inLen - gramSize;
int end = start + gramSize;
termAtt.setTermBuffer(inStr, start, gramSize);
termAtt.setEmpty().append(inStr, start, end);
offsetAtt.setOffset(correctOffset(start), correctOffset(end));
gramSize++;
return true;

15
modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
View File

@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* Tokenizes the input into n-grams of the given size(s).
@ -39,8 +39,8 @@ public final class NGramTokenFilter extends TokenFilter {
private int curPos;
private int tokStart;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Creates NGramTokenFilter with given min and max n-grams.
@ -58,9 +58,6 @@ public final class NGramTokenFilter extends TokenFilter {
}
this.minGram = minGram;
this.maxGram = maxGram;
this.termAtt = addAttribute(TermAttribute.class);
this.offsetAtt = addAttribute(OffsetAttribute.class);
}
/**
@ -79,8 +76,8 @@ public final class NGramTokenFilter extends TokenFilter {
if (!input.incrementToken()) {
return false;
} else {
curTermBuffer = termAtt.termBuffer().clone();
curTermLength = termAtt.termLength();
curTermBuffer = termAtt.buffer().clone();
curTermLength = termAtt.length();
curGramSize = minGram;
curPos = 0;
tokStart = offsetAtt.startOffset();
@ -89,7 +86,7 @@ public final class NGramTokenFilter extends TokenFilter {
while (curGramSize <= maxGram) {
while (curPos+curGramSize <= curTermLength) { // while there is input
clearAttributes();
termAtt.setTermBuffer(curTermBuffer, curPos, curGramSize);
termAtt.copyBuffer(curTermBuffer, curPos, curGramSize);
offsetAtt.setOffset(tokStart + curPos, tokStart + curPos + curGramSize);
curPos++;
return true;

11
modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
View File

@ -18,8 +18,8 @@ package org.apache.lucene.analysis.ngram;
*/
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.AttributeSource;
import java.io.IOException;
@ -39,8 +39,8 @@ public final class NGramTokenizer extends Tokenizer {
private String inStr;
private boolean started = false;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Creates NGramTokenizer with given min and max n-grams.
@ -94,9 +94,6 @@ public final class NGramTokenizer extends Tokenizer {
}
this.minGram = minGram;
this.maxGram = maxGram;
this.termAtt = addAttribute(TermAttribute.class);
this.offsetAtt = addAttribute(OffsetAttribute.class);
}
/** Returns the next token in the stream, or null at EOS. */
@ -123,7 +120,7 @@ public final class NGramTokenizer extends Tokenizer {
int oldPos = pos;
pos++;
termAtt.setTermBuffer(inStr, oldPos, gramSize);
termAtt.setEmpty().append(inStr, oldPos, oldPos+gramSize);
offsetAtt.setOffset(correctOffset(oldPos), correctOffset(oldPos+gramSize));
return true;
}

15
modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java
View File

@ -28,7 +28,7 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* A {@link TokenFilter} that stems Dutch words.
@ -52,17 +52,14 @@ public final class DutchStemFilter extends TokenFilter {
/**
* The actual token in the input stream.
*/
private DutchStemmer stemmer = null;
private DutchStemmer stemmer = new DutchStemmer();
private Set<?> exclusions = null;
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
public DutchStemFilter(TokenStream _in) {
super(_in);
stemmer = new DutchStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
}
/**
@ -99,14 +96,14 @@ public final class DutchStemFilter extends TokenFilter {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
final String term = termAtt.term();
final String term = termAtt.toString();
// Check the exclusion table.
if (!keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains(term))) {
final String s = stemmer.stem(term);
// If not stemmed, don't waste the time adjusting the token.
if ((s != null) && !s.equals(term))
termAtt.setTermBuffer(s);
termAtt.setEmpty().append(s);
}
return true;
} else {

14
modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java
View File

@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
@ -39,15 +39,13 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
public final class DelimitedPayloadTokenFilter extends TokenFilter {
public static final char DEFAULT_DELIMITER = '|';
private final char delimiter;
private final TermAttribute termAtt;
private final PayloadAttribute payAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PayloadAttribute payAtt = addAttribute(PayloadAttribute.class);
private final PayloadEncoder encoder;
public DelimitedPayloadTokenFilter(TokenStream input, char delimiter, PayloadEncoder encoder) {
super(input);
termAtt = addAttribute(TermAttribute.class);
payAtt = addAttribute(PayloadAttribute.class);
this.delimiter = delimiter;
this.encoder = encoder;
}
@ -55,12 +53,12 @@ public final class DelimitedPayloadTokenFilter extends TokenFilter {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
final char[] buffer = termAtt.termBuffer();
final int length = termAtt.termLength();
final char[] buffer = termAtt.buffer();
final int length = termAtt.length();
for (int i = 0; i < length; i++) {
if (buffer[i] == delimiter) {
payAtt.setPayload(encoder.encode(buffer, i + 1, (length - (i + 1))));
termAtt.setTermLength(i); // simply set a new length
termAtt.setLength(i); // simply set a new length
return true;
}
}

6
modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
View File

@ -35,16 +35,14 @@ public class NumericPayloadTokenFilter extends TokenFilter {
private String typeMatch;
private Payload thePayload;
private PayloadAttribute payloadAtt;
private TypeAttribute typeAtt;
private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch) {
super(input);
//Need to encode the payload
thePayload = new Payload(PayloadHelper.encodeFloat(payload));
this.typeMatch = typeMatch;
payloadAtt = addAttribute(PayloadAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}
@Override

6
modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
View File

@ -33,13 +33,11 @@ import org.apache.lucene.index.Payload;
*
**/
public class TokenOffsetPayloadTokenFilter extends TokenFilter {
protected OffsetAttribute offsetAtt;
protected PayloadAttribute payAtt;
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final PayloadAttribute payAtt = addAttribute(PayloadAttribute.class);
public TokenOffsetPayloadTokenFilter(TokenStream input) {
super(input);
offsetAtt = addAttribute(OffsetAttribute.class);
payAtt = addAttribute(PayloadAttribute.class);
}
@Override

6
modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
View File

@ -33,13 +33,11 @@ import java.io.IOException;
*
**/
public class TypeAsPayloadTokenFilter extends TokenFilter {
private PayloadAttribute payloadAtt;
private TypeAttribute typeAtt;
private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
public TypeAsPayloadTokenFilter(TokenStream input) {
super(input);
payloadAtt = addAttribute(PayloadAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}

3
modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
View File

@ -35,7 +35,7 @@ public final class PositionFilter extends TokenFilter {
/** The first token must have non-zero positionIncrement **/
private boolean firstTokenPositioned = false;
private PositionIncrementAttribute posIncrAtt;
private PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
/**
* Constructs a PositionFilter that assigns a position increment of zero to
@ -45,7 +45,6 @@ public final class PositionFilter extends TokenFilter {
*/
public PositionFilter(final TokenStream input) {
super(input);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
/**

15
modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.analysis.reverse;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
import java.io.IOException;
@ -42,7 +42,7 @@ import java.io.IOException;
*/
public final class ReverseStringFilter extends TokenFilter {
private TermAttribute termAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final char marker;
private final Version matchVersion;
private static final char NOMARKER = '\uFFFF';
@ -131,20 +131,19 @@ public final class ReverseStringFilter extends TokenFilter {
super(in);
this.matchVersion = matchVersion;
this.marker = marker;
termAtt = addAttribute(TermAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
int len = termAtt.termLength();
int len = termAtt.length();
if (marker != NOMARKER) {
len++;
termAtt.resizeTermBuffer(len);
termAtt.termBuffer()[len - 1] = marker;
termAtt.resizeBuffer(len);
termAtt.buffer()[len - 1] = marker;
}
reverse( matchVersion, termAtt.termBuffer(), 0, len );
termAtt.setTermLength(len);
reverse( matchVersion, termAtt.buffer(), 0, len );
termAtt.setLength(len);
return true;
} else {
return false;

9
modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java
View File

@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* Normalizes token text to lower case.
@ -32,20 +32,19 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
@Deprecated
public final class RussianLowerCaseFilter extends TokenFilter
{
private TermAttribute termAtt;
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public RussianLowerCaseFilter(TokenStream in)
{
super(in);
termAtt = addAttribute(TermAttribute.class);
}
@Override
public final boolean incrementToken() throws IOException
{
if (input.incrementToken()) {
char[] chArray = termAtt.termBuffer();
int chLen = termAtt.termLength();
char[] chArray = termAtt.buffer();
int chLen = termAtt.length();
for (int i = 0; i < chLen; i++)
{
chArray[i] = Character.toLowerCase(chArray[i]);

15
modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; // for java
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.ru.RussianStemmer;//javadoc @link
import org.apache.lucene.analysis.snowball.SnowballFilter; // javadoc @link
@ -51,17 +51,14 @@ public final class RussianStemFilter extends TokenFilter
/**
* The actual token in the input stream.
*/
private RussianStemmer stemmer = null;
private RussianStemmer stemmer = new RussianStemmer();
private final TermAttribute termAtt;
private final KeywordAttribute keywordAttr;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
public RussianStemFilter(TokenStream in)
{
super(in);
stemmer = new RussianStemmer();
termAtt = addAttribute(TermAttribute.class);
keywordAttr = addAttribute(KeywordAttribute.class);
}
/**
* Returns the next token in the stream, or null at EOS
@ -71,10 +68,10 @@ public final class RussianStemFilter extends TokenFilter
{
if (input.incrementToken()) {
if(!keywordAttr.isKeyword()) {
final String term = termAtt.term();
final String term = termAtt.toString();
final String s = stemmer.stem(term);
if (s != null && !s.equals(term))
termAtt.setTermBuffer(s);
termAtt.setEmpty().append(s);
}
return true;
} else {

12
modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
View File

@ -137,10 +137,10 @@ public final class ShingleFilter extends TokenFilter {
*/
private boolean isOutputHere = false;
private final CharTermAttribute termAtt;
private final OffsetAttribute offsetAtt;
private final PositionIncrementAttribute posIncrAtt;
private final TypeAttribute typeAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
/**
@ -155,10 +155,6 @@ public final class ShingleFilter extends TokenFilter {
super(input);
setMaxShingleSize(maxShingleSize);
setMinShingleSize(minShingleSize);
this.termAtt = addAttribute(CharTermAttribute.class);
this.offsetAtt = addAttribute(OffsetAttribute.class);
this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
this.typeAtt = addAttribute(TypeAttribute.class);
}
/**

26
modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
View File

@ -31,11 +31,11 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.EmptyTokenStream;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix.Column.Row;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.index.Payload;
@ -193,14 +193,14 @@ public final class ShingleMatrixFilter extends TokenStream {
private TokenStream input;
private TermAttribute termAtt;
private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncrAtt;
private PayloadAttribute payloadAtt;
private OffsetAttribute offsetAtt;
private TypeAttribute typeAtt;
private FlagsAttribute flagsAtt;
private TermAttribute in_termAtt;
private CharTermAttribute in_termAtt;
private PositionIncrementAttribute in_posIncrAtt;
private PayloadAttribute in_payloadAtt;
private OffsetAttribute in_offsetAtt;
@ -229,7 +229,7 @@ public final class ShingleMatrixFilter extends TokenStream {
this.ignoringSinglePrefixOrSuffixShingle = ignoringSinglePrefixOrSuffixShingle;
this.settingsCodec = settingsCodec;
termAtt = addAttribute(TermAttribute.class);
termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
@ -239,7 +239,7 @@ public final class ShingleMatrixFilter extends TokenStream {
// set the input to be an empty token stream, we already have the data.
this.input = new EmptyTokenStream();
in_termAtt = input.addAttribute(TermAttribute.class);
in_termAtt = input.addAttribute(CharTermAttribute.class);
in_posIncrAtt = input.addAttribute(PositionIncrementAttribute.class);
in_payloadAtt = input.addAttribute(PayloadAttribute.class);
in_offsetAtt = input.addAttribute(OffsetAttribute.class);
@ -311,14 +311,14 @@ public final class ShingleMatrixFilter extends TokenStream {
this.spacerCharacter = spacerCharacter;
this.ignoringSinglePrefixOrSuffixShingle = ignoringSinglePrefixOrSuffixShingle;
this.settingsCodec = settingsCodec;
termAtt = addAttribute(TermAttribute.class);
termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
flagsAtt = addAttribute(FlagsAttribute.class);
in_termAtt = input.addAttribute(TermAttribute.class);
in_termAtt = input.addAttribute(CharTermAttribute.class);
in_posIncrAtt = input.addAttribute(PositionIncrementAttribute.class);
in_payloadAtt = input.addAttribute(PayloadAttribute.class);
in_offsetAtt = input.addAttribute(OffsetAttribute.class);
@ -377,7 +377,7 @@ public final class ShingleMatrixFilter extends TokenStream {
if (token == null) return false;
clearAttributes();
termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
termAtt.copyBuffer(token.buffer(), 0, token.length());
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
flagsAtt.setFlags(token.getFlags());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
@ -388,7 +388,7 @@ public final class ShingleMatrixFilter extends TokenStream {
private Token getNextInputToken(Token token) throws IOException {
if (!input.incrementToken()) return null;
token.setTermBuffer(in_termAtt.termBuffer(), 0, in_termAtt.termLength());
token.copyBuffer(in_termAtt.buffer(), 0, in_termAtt.length());
token.setPositionIncrement(in_posIncrAtt.getPositionIncrement());
token.setFlags(in_flagsAtt.getFlags());
token.setOffset(in_offsetAtt.startOffset(), in_offsetAtt.endOffset());
@ -399,7 +399,7 @@ public final class ShingleMatrixFilter extends TokenStream {
private Token getNextToken(Token token) throws IOException {
if (!this.incrementToken()) return null;
token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
token.setPositionIncrement(posIncrAtt.getPositionIncrement());
token.setFlags(flagsAtt.getFlags());
token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
@ -441,7 +441,7 @@ public final class ShingleMatrixFilter extends TokenStream {
for (int i = 0; i < currentShingleLength; i++) {
Token shingleToken = currentPermuationTokens.get(i + currentPermutationTokensStartOffset);
termLength += shingleToken.termLength();
termLength += shingleToken.length();
shingle.add(shingleToken);
}
if (spacerCharacter != null) {
@ -459,9 +459,9 @@ public final class ShingleMatrixFilter extends TokenStream {
if (spacerCharacter != null && sb.length() > 0) {
sb.append(spacerCharacter);
}
sb.append(shingleToken.termBuffer(), 0, shingleToken.termLength());
sb.append(shingleToken.buffer(), 0, shingleToken.length());
}
reusableToken.setTermBuffer(sb.toString());
reusableToken.setEmpty().append(sb);
updateToken(reusableToken, shingle, currentPermutationTokensStartOffset, currentPermutationRows, currentPermuationTokens);
return reusableToken;

8
modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java
View File

@ -21,7 +21,7 @@ import java.text.DateFormat;
import java.text.ParseException;
import java.util.Date;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.AttributeSource;
/**
@ -34,7 +34,7 @@ public class DateRecognizerSinkFilter extends TeeSinkTokenFilter.SinkFilter {
public static final String DATE_TYPE = "date";
protected DateFormat dateFormat;
protected TermAttribute termAtt;
protected CharTermAttribute termAtt;
/**
* Uses {@link java.text.SimpleDateFormat#getDateInstance()} as the {@link java.text.DateFormat} object.
@ -50,10 +50,10 @@ public class DateRecognizerSinkFilter extends TeeSinkTokenFilter.SinkFilter {
@Override
public boolean accept(AttributeSource source) {
if (termAtt == null) {
termAtt = source.addAttribute(TermAttribute.class);
termAtt = source.addAttribute(CharTermAttribute.class);
}
try {
Date date = dateFormat.parse(termAtt.term());//We don't care about the date, just that we can parse it as a date
Date date = dateFormat.parse(termAtt.toString());//We don't care about the date, just that we can parse it as a date
if (date != null) {
return true;
}

12
modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tr.TurkishLowerCaseFilter; // javadoc @link
import org.tartarus.snowball.SnowballProgram;
@ -42,7 +42,7 @@ public final class SnowballFilter extends TokenFilter {
private final SnowballProgram stemmer;
private final TermAttribute termAtt = addAttribute(TermAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
public SnowballFilter(TokenStream input, SnowballProgram stemmer) {
@ -76,16 +76,16 @@ public final class SnowballFilter extends TokenFilter {
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAttr.isKeyword()) {
char termBuffer[] = termAtt.termBuffer();
final int length = termAtt.termLength();
char termBuffer[] = termAtt.buffer();
final int length = termAtt.length();
stemmer.setCurrent(termBuffer, length);
stemmer.stem();
final char finalTerm[] = stemmer.getCurrentBuffer();
final int newLength = stemmer.getCurrentBufferLength();
if (finalTerm != termBuffer)
termAtt.setTermBuffer(finalTerm, 0, newLength);
termAtt.copyBuffer(finalTerm, 0, newLength);
else
termAtt.setTermLength(newLength);
termAtt.setLength(newLength);
}
return true;
} else {

2
modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
View File

@ -133,7 +133,7 @@ public final class SynonymFilter extends TokenFilter {
OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);
newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
newTermAtt.copyBuffer(repTok.termBuffer(), 0, repTok.termLength());
newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
repPos += repTok.getPositionIncrement();
if (i==0) repPos=origPos; // make position of first token equal to original

7
modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
View File

@ -103,8 +103,7 @@ public class SynonymMap {
List<Token> ret = new ArrayList<Token>(strings.size());
for (String str : strings) {
//Token newTok = new Token(str,0,0,"SYNONYM");
Token newTok = new Token(0,0,"SYNONYM");
newTok.setTermBuffer(str.toCharArray(), 0, str.length());
Token newTok = new Token(str, 0,0,"SYNONYM");
ret.add(newTok);
}
return ret;
@ -137,7 +136,7 @@ public class SynonymMap {
while(tok1!=null || tok2!=null) {
while (tok1 != null && (pos1 <= pos2 || tok2==null)) {
Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type());
tok.setTermBuffer(tok1.termBuffer(), 0, tok1.termLength());
tok.copyBuffer(tok1.buffer(), 0, tok1.length());
tok.setPositionIncrement(pos1-pos);
result.add(tok);
pos=pos1;
@ -146,7 +145,7 @@ public class SynonymMap {
}
while (tok2 != null && (pos2 <= pos1 || tok1==null)) {
Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type());
tok.setTermBuffer(tok2.termBuffer(), 0, tok2.termLength());
tok.copyBuffer(tok2.buffer(), 0, tok2.length());
tok.setPositionIncrement(pos2-pos);
result.add(tok);
pos=pos2;

11
modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java
View File

@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* Normalizes Turkish token text to lower case.
@ -37,7 +37,7 @@ public final class TurkishLowerCaseFilter extends TokenFilter {
private static final int LATIN_SMALL_LETTER_I = '\u0069';
private static final int LATIN_SMALL_LETTER_DOTLESS_I = '\u0131';
private static final int COMBINING_DOT_ABOVE = '\u0307';
private final TermAttribute termAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
/**
* Create a new TurkishLowerCaseFilter, that normalizes Turkish token text
@ -47,7 +47,6 @@ public final class TurkishLowerCaseFilter extends TokenFilter {
*/
public TurkishLowerCaseFilter(TokenStream in) {
super(in);
termAtt = addAttribute(TermAttribute.class);
}
@Override
@ -55,8 +54,8 @@ public final class TurkishLowerCaseFilter extends TokenFilter {
boolean iOrAfter = false;
if (input.incrementToken()) {
final char[] buffer = termAtt.termBuffer();
int length = termAtt.termLength();
final char[] buffer = termAtt.buffer();
int length = termAtt.length();
for (int i = 0; i < length;) {
final int ch = Character.codePointAt(buffer, i);
@ -88,7 +87,7 @@ public final class TurkishLowerCaseFilter extends TokenFilter {
i += Character.toChars(Character.toLowerCase(ch), buffer, i);
}
termAtt.setTermLength(length);
termAtt.setLength(length);
return true;
} else
return false;

27
modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
View File

@ -18,10 +18,10 @@
package org.apache.lucene.analysis.wikipedia;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;
@ -116,11 +116,11 @@ public final class WikipediaTokenizer extends Tokenizer {
private Set<String> untokenizedTypes = Collections.emptySet();
private Iterator<AttributeSource.State> tokens = null;
private OffsetAttribute offsetAtt;
private TypeAttribute typeAtt;
private PositionIncrementAttribute posIncrAtt;
private TermAttribute termAtt;
private FlagsAttribute flagsAtt;
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
/**
* Creates a new instance of the {@link WikipediaTokenizer}. Attaches the
@ -176,12 +176,7 @@ public final class WikipediaTokenizer extends Tokenizer {
private void init(int tokenOutput, Set<String> untokenizedTypes) {
this.tokenOutput = tokenOutput;
this.untokenizedTypes = untokenizedTypes;
this.offsetAtt = addAttribute(OffsetAttribute.class);
this.typeAtt = addAttribute(TypeAttribute.class);
this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
this.termAtt = addAttribute(TermAttribute.class);
this.flagsAtt = addAttribute(FlagsAttribute.class);
this.untokenizedTypes = untokenizedTypes;
}
/*
@ -245,8 +240,9 @@ public final class WikipediaTokenizer extends Tokenizer {
lastPos = currPos + numAdded;
}
//trim the buffer
// TODO: this is inefficient
String s = buffer.toString().trim();
termAtt.setTermBuffer(s.toCharArray(), 0, s.length());
termAtt.setEmpty().append(s);
offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
//The way the loop is written, we will have proceeded to the next token. We need to pushback the scanner to lastPos
@ -283,8 +279,9 @@ public final class WikipediaTokenizer extends Tokenizer {
lastPos = currPos + numAdded;
}
//trim the buffer
// TODO: this is inefficient
String s = buffer.toString().trim();
termAtt.setTermBuffer(s.toCharArray(), 0, s.length());
termAtt.setEmpty().append(s);
offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
//The way the loop is written, we will have proceeded to the next token. We need to pushback the scanner to lastPos
@ -298,7 +295,7 @@ public final class WikipediaTokenizer extends Tokenizer {
private void setupToken() {
scanner.getText(termAtt);
final int start = scanner.yychar();
offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.termLength()));
offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));
}
/*

324
modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
View File

@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:51 */
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 5/31/10 3:11 PM */
package org.apache.lucene.analysis.wikipedia;
@ -19,14 +19,14 @@ package org.apache.lucene.analysis.wikipedia;
* limitations under the License.
*/
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
* on 17.05.10 14:51 from the specification file
* <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
* on 5/31/10 3:11 PM from the specification file
* <tt>C:/Users/rmuir/workspace/solrcene/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
*/
class WikipediaTokenizerImpl {
@ -37,16 +37,16 @@ class WikipediaTokenizerImpl {
private static final int ZZ_BUFFERSIZE = 16384;
/** lexical states */
public static final int CATEGORY_STATE = 2;
public static final int DOUBLE_EQUALS_STATE = 14;
public static final int THREE_SINGLE_QUOTES_STATE = 10;
public static final int EXTERNAL_LINK_STATE = 6;
public static final int DOUBLE_EQUALS_STATE = 14;
public static final int INTERNAL_LINK_STATE = 4;
public static final int DOUBLE_BRACE_STATE = 16;
public static final int FIVE_SINGLE_QUOTES_STATE = 12;
public static final int STRING = 18;
public static final int TWO_SINGLE_QUOTES_STATE = 8;
public static final int CATEGORY_STATE = 2;
public static final int YYINITIAL = 0;
public static final int THREE_SINGLE_QUOTES_STATE = 10;
public static final int STRING = 18;
public static final int FIVE_SINGLE_QUOTES_STATE = 12;
public static final int TWO_SINGLE_QUOTES_STATE = 8;
/**
* ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
@ -487,8 +487,8 @@ public final int getPositionIncrement(){
/**
* Fills Lucene token with the current token text.
*/
final void getText(TermAttribute t) {
t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
final void getText(CharTermAttribute t) {
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}
final int setText(StringBuilder buffer){
@ -803,184 +803,184 @@ final int setText(StringBuilder buffer){
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
case 25:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);
}
case 46: break;
case 30:
{ numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/
}
case 47: break;
case 41:
{ numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/
}
case 48: break;
case 14:
{ yybegin(STRING); numWikiTokensSeen++; return currentTokType;
}
case 49: break;
case 23:
{ numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);
}
case 50: break;
case 34:
{ positionInc = 1; return NUM;
}
case 51: break;
case 18:
{ /* ignore STRING */
}
case 52: break;
case 12:
{ currentTokType = ITALICS; numWikiTokensSeen++; yybegin(STRING); return currentTokType;/*italics*/
}
case 53: break;
case 37:
{ numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/
}
case 54: break;
case 31:
{ numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);
}
case 55: break;
case 10:
{ numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);
}
case 56: break;
case 38:
{ numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/
}
case 57: break;
case 19:
{ yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
}
case 58: break;
case 11:
{ currentTokType = BOLD; yybegin(THREE_SINGLE_QUOTES_STATE);
}
case 59: break;
case 1:
{ numWikiTokensSeen = 0; positionInc = 1;
}
case 60: break;
case 33:
{ positionInc = 1; return HOST;
}
case 61: break;
case 3:
{ positionInc = 1; return CJ;
}
case 62: break;
case 17:
{ yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
}
case 63: break;
case 32:
{ positionInc = 1; return APOSTROPHE;
}
case 64: break;
case 8:
{ /* ignore */
}
case 65: break;
case 4:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);
}
case 66: break;
case 2:
{ positionInc = 1; return ALPHANUM;
}
case 67: break;
case 26:
{ yybegin(YYINITIAL);
}
case 68: break;
case 43:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);
}
case 69: break;
case 36:
{ currentTokType = BOLD_ITALICS; yybegin(FIVE_SINGLE_QUOTES_STATE);
}
case 70: break;
case 13:
{ currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);
}
case 71: break;
case 24:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);
}
case 72: break;
case 27:
{ numLinkToks = 0; yybegin(YYINITIAL);
}
case 73: break;
case 15:
{ currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);
}
case 74: break;
case 28:
{ currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
}
case 75: break;
case 39:
{ positionInc = 1; return ACRONYM;
}
case 76: break;
case 29:
{ currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
}
case 77: break;
case 7:
{ yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
}
case 78: break;
case 16:
{ currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;
}
case 79: break;
case 46: break;
case 39:
{ positionInc = 1; return ACRONYM;
}
case 47: break;
case 8:
{ /* ignore */
}
case 48: break;
case 20:
{ numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE);
}
case 80: break;
case 49: break;
case 35:
{ positionInc = 1; return COMPANY;
}
case 50: break;
case 4:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);
}
case 51: break;
case 25:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);
}
case 52: break;
case 43:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);
}
case 53: break;
case 22:
{ numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}
}
case 54: break;
case 34:
{ positionInc = 1; return NUM;
}
case 55: break;
case 32:
{ positionInc = 1; return APOSTROPHE;
}
case 56: break;
case 23:
{ numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);
}
case 57: break;
case 21:
{ yybegin(STRING); return currentTokType;/*pipe*/
}
case 58: break;
case 2:
{ positionInc = 1; return ALPHANUM;
}
case 59: break;
case 29:
{ currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
}
case 60: break;
case 17:
{ yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
}
case 61: break;
case 44:
{ currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);
}
case 62: break;
case 26:
{ yybegin(YYINITIAL);
}
case 63: break;
case 3:
{ positionInc = 1; return CJ;
}
case 64: break;
case 38:
{ numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/
}
case 65: break;
case 15:
{ currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);
}
case 66: break;
case 30:
{ numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/
}
case 67: break;
case 6:
{ yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
}
case 68: break;
case 5:
{ positionInc = 1;
}
case 69: break;
case 19:
{ yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
}
case 70: break;
case 42:
{ positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
}
case 71: break;
case 27:
{ numLinkToks = 0; yybegin(YYINITIAL);
}
case 72: break;
case 11:
{ currentTokType = BOLD; yybegin(THREE_SINGLE_QUOTES_STATE);
}
case 73: break;
case 13:
{ currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);
}
case 74: break;
case 14:
{ yybegin(STRING); numWikiTokensSeen++; return currentTokType;
}
case 75: break;
case 45:
{ numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);
}
case 76: break;
case 28:
{ currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
}
case 77: break;
case 37:
{ numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/
}
case 78: break;
case 9:
{ if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
}
case 79: break;
case 7:
{ yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
}
case 80: break;
case 24:
{ numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);
}
case 81: break;
case 40:
{ positionInc = 1; return EMAIL;
}
case 82: break;
case 42:
{ positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
case 1:
{ numWikiTokensSeen = 0; positionInc = 1;
}
case 83: break;
case 6:
{ yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
case 18:
{ /* ignore STRING */
}
case 84: break;
case 44:
{ currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);
case 36:
{ currentTokType = BOLD_ITALICS; yybegin(FIVE_SINGLE_QUOTES_STATE);
}
case 85: break;
case 5:
{ positionInc = 1;
case 33:
{ positionInc = 1; return HOST;
}
case 86: break;
case 9:
{ if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
case 31:
{ numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);
}
case 87: break;
case 45:
{ numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);
case 41:
{ numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/
}
case 88: break;
case 22:
{ numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}
case 12:
{ currentTokType = ITALICS; numWikiTokensSeen++; yybegin(STRING); return currentTokType;/*italics*/
}
case 89: break;
case 21:
{ yybegin(STRING); return currentTokType;/*pipe*/
case 10:
{ numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);
}
case 90: break;
default:

6
modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
View File

@ -17,7 +17,7 @@ package org.apache.lucene.analysis.wikipedia;
* limitations under the License.
*/
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
%%
@ -81,8 +81,8 @@ public final int getPositionIncrement(){
/**
* Fills Lucene token with the current token text.
*/
final void getText(TermAttribute t) {
t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
final void getText(CharTermAttribute t) {
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}
final int setText(StringBuilder buffer){

12
modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
View File

@ -17,8 +17,6 @@ package org.apache.lucene.analysis.compound;
* limitations under the License.
*/
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
@ -27,7 +25,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase {
public void testHyphenationCompoundWordsDA() throws Exception {
@ -176,15 +174,15 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase {
CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false);
TermAttribute termAtt = tf.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = tf.getAttribute(CharTermAttribute.class);
assertTrue(tf.incrementToken());
assertEquals("Rindfleischüberwachungsgesetz", termAtt.term());
assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
assertTrue(tf.incrementToken());
assertEquals("Rind", termAtt.term());
assertEquals("Rind", termAtt.toString());
wsTokenizer.reset(new StringReader("Rindfleischüberwachungsgesetz"));
tf.reset();
assertTrue(tf.incrementToken());
assertEquals("Rindfleischüberwachungsgesetz", termAtt.term());
assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
}
private Reader getHyphenationReader() throws Exception {

5
modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java
View File

@ -28,6 +28,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
/**
@ -50,9 +51,9 @@ public class TestElision extends BaseTokenStreamTestCase {
private List<String> filter(TokenFilter filter) throws IOException {
List<String> tas = new ArrayList<String>();
TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
while (filter.incrementToken()) {
tas.add(termAtt.term());
tas.add(termAtt.toString());
}
return tas;
}

4
modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java
View File

@ -41,8 +41,6 @@ public class TestPrefixAndSuffixAwareTokenFilter extends BaseTokenStreamTestCase
private static Token createToken(String term, int start, int offset)
{
Token token = new Token(start, offset);
token.setTermBuffer(term);
return token;
return new Token(term, start, offset);
}
}

4
modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java
View File

@ -52,8 +52,6 @@ public class TestPrefixAwareTokenFilter extends BaseTokenStreamTestCase {
private static Token createToken(String term, int start, int offset)
{
Token token = new Token(start, offset);
token.setTermBuffer(term);
return token;
return new Token(term, start, offset);
}
}

2
modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java
View File

@ -51,7 +51,7 @@ public class TestRemoveDuplicatesTokenFilter extends BaseTokenStreamTestCase {
if (toks.hasNext()) {
clearAttributes();
Token tok = toks.next();
termAtt.setEmpty().append(tok.term());
termAtt.setEmpty().append(tok);
offsetAtt.setOffset(tok.startOffset(), tok.endOffset());
posIncAtt.setPositionIncrement(tok.getPositionIncrement());
return true;

4
modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
View File

@ -22,14 +22,14 @@ import java.io.IOException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class TestSingleTokenTokenFilter extends LuceneTestCase {
public void test() throws IOException {
Token token = new Token();
SingleTokenTokenStream ts = new SingleTokenTokenStream(token);
AttributeImpl tokenAtt = (AttributeImpl) ts.addAttribute(TermAttribute.class);
AttributeImpl tokenAtt = (AttributeImpl) ts.addAttribute(CharTermAttribute.class);
assertTrue(tokenAtt instanceof Token);
ts.reset();

2
modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
View File

@ -97,7 +97,7 @@ public class TestTrimFilter extends BaseTokenStreamTestCase {
else {
clearAttributes();
Token token = tokens[index++];
termAtt.setEmpty().append(token.term());
termAtt.setEmpty().append(token);
offsetAtt.setOffset(token.startOffset(), token.endOffset());
posIncAtt.setPositionIncrement(token.getPositionIncrement());
flagsAtt.setFlags(token.getFlags());

16
modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
View File

@ -18,8 +18,8 @@ package org.apache.lucene.analysis.payloads;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.Payload;
import org.apache.lucene.util.LuceneTestCase;
@ -32,7 +32,7 @@ public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test)),
DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
assertTermEquals("The", filter, termAtt, payAtt, null);
assertTermEquals("quick", filter, termAtt, payAtt, "JJ".getBytes("UTF-8"));
@ -70,7 +70,7 @@ public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {
public void testFloatEncoding() throws Exception {
String test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test)), '|', new FloatEncoder());
TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
assertTermEquals("The", filter, termAtt, payAtt, null);
assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeFloat(1.0f));
@ -88,7 +88,7 @@ public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {
public void testIntEncoding() throws Exception {
String test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test)), '|', new IntegerEncoder());
TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
assertTermEquals("The", filter, termAtt, payAtt, null);
assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeInt(1));
@ -104,10 +104,10 @@ public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {
}
void assertTermEquals(String expected, TokenStream stream, byte[] expectPay) throws Exception {
TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
PayloadAttribute payloadAtt = stream.getAttribute(PayloadAttribute.class);
assertTrue(stream.incrementToken());
assertEquals(expected, termAtt.term());
assertEquals(expected, termAtt.toString());
Payload payload = payloadAtt.getPayload();
if (payload != null) {
assertTrue(payload.length() + " does not equal: " + expectPay.length, payload.length() == expectPay.length);
@ -121,9 +121,9 @@ public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {
}
void assertTermEquals(String expected, TokenStream stream, TermAttribute termAtt, PayloadAttribute payAtt, byte[] expectPay) throws Exception {
void assertTermEquals(String expected, TokenStream stream, CharTermAttribute termAtt, PayloadAttribute payAtt, byte[] expectPay) throws Exception {
assertTrue(stream.incrementToken());
assertEquals(expected, termAtt.term());
assertEquals(expected, termAtt.toString());
Payload payload = payAtt.getPayload();
if (payload != null) {
assertTrue(payload.length() + " does not equal: " + expectPay.length, payload.length() == expectPay.length);

14
modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
View File

@ -20,8 +20,8 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import java.io.IOException;
@ -39,11 +39,11 @@ public class NumericPayloadTokenFilterTest extends BaseTokenStreamTestCase {
NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test))), 3, "D");
boolean seenDogs = false;
TermAttribute termAtt = nptf.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
PayloadAttribute payloadAtt = nptf.getAttribute(PayloadAttribute.class);
while (nptf.incrementToken()) {
if (termAtt.term().equals("dogs")) {
if (termAtt.toString().equals("dogs")) {
seenDogs = true;
assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().equals("D") == true);
assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.getPayload() != null);
@ -60,19 +60,17 @@ public class NumericPayloadTokenFilterTest extends BaseTokenStreamTestCase {
}
private final class WordTokenFilter extends TokenFilter {
private TermAttribute termAtt;
private TypeAttribute typeAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private WordTokenFilter(TokenStream input) {
super(input);
termAtt = addAttribute(TermAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (termAtt.term().equals("dogs"))
if (termAtt.toString().equals("dogs"))
typeAtt.setType("D");
return true;
} else {

14
modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
View File

@ -21,7 +21,7 @@ import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import java.io.IOException;
@ -39,12 +39,12 @@ public class TypeAsPayloadTokenFilterTest extends BaseTokenStreamTestCase {
TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test))));
int count = 0;
TermAttribute termAtt = nptf.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
PayloadAttribute payloadAtt = nptf.getAttribute(PayloadAttribute.class);
while (nptf.incrementToken()) {
assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0]))));
assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals(String.valueOf(Character.toUpperCase(termAtt.buffer()[0]))));
assertTrue("nextToken.getPayload() is null and it shouldn't be", payloadAtt.getPayload() != null);
String type = new String(payloadAtt.getPayload().getData(), "UTF-8");
assertTrue(type + " is not equal to " + typeAtt.type(), type.equals(typeAtt.type()) == true);
@ -55,19 +55,17 @@ public class TypeAsPayloadTokenFilterTest extends BaseTokenStreamTestCase {
}
private final class WordTokenFilter extends TokenFilter {
private TermAttribute termAtt;
private TypeAttribute typeAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private WordTokenFilter(TokenStream input) {
super(input);
termAtt = addAttribute(TermAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
typeAtt.setType(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0])));
typeAtt.setType(String.valueOf(Character.toUpperCase(termAtt.buffer()[0])));
return true;
} else {
return false;

7
modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java
View File

@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.shingle.ShingleFilter;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class PositionFilterTest extends BaseTokenStreamTestCase {
@ -30,19 +30,18 @@ public class PositionFilterTest extends BaseTokenStreamTestCase {
protected int index = 0;
protected String[] testToken;
protected TermAttribute termAtt;
protected final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public TestTokenStream(String[] testToken) {
super();
this.testToken = testToken;
termAtt = addAttribute(TermAttribute.class);
}
@Override
public final boolean incrementToken() throws IOException {
clearAttributes();
if (index < testToken.length) {
termAtt.setTermBuffer(testToken[index++]);
termAtt.setEmpty().append(testToken[index++]);
return true;
} else {
return false;

6
modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
View File

@ -26,7 +26,6 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@ -176,9 +175,6 @@ public class QueryAutoStopWordAnalyzerTest extends BaseTokenStreamTestCase {
QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
a.addStopWords(reader, 10);
TokenStream ts = a.tokenStream("repetitiveField", new StringReader("this boring"));
TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
assertTrue(ts.incrementToken());
assertEquals("this", termAtt.term());
assertFalse(ts.incrementToken());
assertTokenStreamContents(ts, new String[] { "this" });
}
}

30
modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
View File

@ -21,46 +21,22 @@ import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.util.Version;
public class TestReverseStringFilter extends BaseTokenStreamTestCase {
public void testFilter() throws Exception {
TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
new StringReader("Do have a nice day")); // 1-4 length string
ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream);
TermAttribute text = filter.getAttribute(TermAttribute.class);
assertTrue(filter.incrementToken());
assertEquals("oD", text.term());
assertTrue(filter.incrementToken());
assertEquals("evah", text.term());
assertTrue(filter.incrementToken());
assertEquals("a", text.term());
assertTrue(filter.incrementToken());
assertEquals("ecin", text.term());
assertTrue(filter.incrementToken());
assertEquals("yad", text.term());
assertFalse(filter.incrementToken());
assertTokenStreamContents(filter, new String[] { "oD", "evah", "a", "ecin", "yad" });
}
public void testFilterWithMark() throws Exception {
TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(
"Do have a nice day")); // 1-4 length string
ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream, '\u0001');
TermAttribute text = filter
.getAttribute(TermAttribute.class);
assertTrue(filter.incrementToken());
assertEquals("\u0001oD", text.term());
assertTrue(filter.incrementToken());
assertEquals("\u0001evah", text.term());
assertTrue(filter.incrementToken());
assertEquals("\u0001a", text.term());
assertTrue(filter.incrementToken());
assertEquals("\u0001ecin", text.term());
assertTrue(filter.incrementToken());
assertEquals("\u0001yad", text.term());
assertFalse(filter.incrementToken());
assertTokenStreamContents(filter,
new String[] { "\u0001oD", "\u0001evah", "\u0001a", "\u0001ecin", "\u0001yad" });
}
public void testReverseString() throws Exception {

35
modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java
View File

@ -17,17 +17,13 @@ package org.apache.lucene.analysis.ru;
* limitations under the License.
*/
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
@ -65,8 +61,8 @@ public class TestRussianAnalyzer extends BaseTokenStreamTestCase
new RussianLetterTokenizer(TEST_VERSION_CURRENT,
sampleUnicode);
TermAttribute text = in.getAttribute(TermAttribute.class);
TermAttribute sampleText = sample.getAttribute(TermAttribute.class);
CharTermAttribute text = in.getAttribute(CharTermAttribute.class);
CharTermAttribute sampleText = sample.getAttribute(CharTermAttribute.class);
for (;;)
{
@ -76,34 +72,21 @@ public class TestRussianAnalyzer extends BaseTokenStreamTestCase
boolean nextSampleToken = sample.incrementToken();
assertEquals(
"Unicode",
text.term(),
text.toString(),
nextSampleToken == false
? null
: sampleText.term());
: sampleText.toString());
}
inWords.close();
sampleUnicode.close();
}
public void testDigitsInRussianCharset()
/** Check that RussianAnalyzer doesnt discard any numbers */
public void testDigitsInRussianCharset() throws IOException
{
Reader reader = new StringReader("text 1000");
RussianAnalyzer ra = new RussianAnalyzer(TEST_VERSION_CURRENT);
TokenStream stream = ra.tokenStream("", reader);
TermAttribute termText = stream.getAttribute(TermAttribute.class);
try {
assertTrue(stream.incrementToken());
assertEquals("text", termText.term());
assertTrue(stream.incrementToken());
assertEquals("RussianAnalyzer's tokenizer skips numbers from input text", "1000", termText.term());
assertFalse(stream.incrementToken());
}
catch (IOException e)
{
fail("unexpected IOException");
}
RussianAnalyzer ra = new RussianAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(ra, "text 1000", new String[] { "text", "1000" });
}
/** @deprecated remove this test in Lucene 4.0: stopwords changed */

10
modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
View File

@ -26,8 +26,8 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
@ -159,11 +159,11 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase {
int j = -1;
PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
j += posIncrAtt.getPositionIncrement();
String termText = termAtt.term();
String termText = termAtt.toString();
q.add(new Term("content", termText), j);
}
@ -186,10 +186,10 @@ public class ShingleAnalyzerWrapperTest extends BaseTokenStreamTestCase {
TokenStream ts = analyzer.tokenStream("content",
new StringReader("test sentence"));
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
String termText = termAtt.term();
String termText = termAtt.toString();
q.add(new TermQuery(new Term("content", termText)),
BooleanClause.Occur.SHOULD);
}

47
modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
View File

@ -31,7 +31,12 @@ import org.apache.lucene.analysis.miscellaneous.SingleTokenTokenStream;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix;
import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix.Column;
import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
@ -415,7 +420,7 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
private Token tokenFactory(String text, int posIncr, int startOffset, int endOffset) {
Token token = new Token(startOffset, endOffset);
token.setTermBuffer(text);
token.setEmpty().append(text);
token.setPositionIncrement(posIncr);
return token;
}
@ -427,7 +432,7 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
private Token tokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset) {
Token token = new Token(startOffset, endOffset);
token.setTermBuffer(text);
token.setEmpty().append(text);
token.setPositionIncrement(posIncr);
ShingleMatrixFilter.defaultSettingsCodec.setWeight(token, weight);
return token;
@ -435,7 +440,7 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
private Token tokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset, ShingleMatrixFilter.TokenPositioner positioner) {
Token token = new Token(startOffset, endOffset);
token.setTermBuffer(text);
token.setEmpty().append(text);
token.setPositionIncrement(posIncr);
ShingleMatrixFilter.defaultSettingsCodec.setWeight(token, weight);
ShingleMatrixFilter.defaultSettingsCodec.setTokenPositioner(token, positioner);
@ -445,20 +450,20 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
// assert-methods start here
private void assertNext(TokenStream ts, String text) throws IOException {
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
assertTrue(ts.incrementToken());
assertEquals(text, termAtt.term());
assertEquals(text, termAtt.toString());
}
private void assertNext(TokenStream ts, String text, int positionIncrement, float boost, int startOffset, int endOffset) throws IOException {
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
assertTrue(ts.incrementToken());
assertEquals(text, termAtt.term());
assertEquals(text, termAtt.toString());
assertEquals(positionIncrement, posIncrAtt.getPositionIncrement());
assertEquals(boost, payloadAtt.getPayload() == null ? 1f : PayloadHelper.decodeFloat(payloadAtt.getPayload().getData()), 0);
assertEquals(startOffset, offsetAtt.startOffset());
@ -466,11 +471,11 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
}
private void assertNext(TokenStream ts, String text, int startOffset, int endOffset) throws IOException {
TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
assertTrue(ts.incrementToken());
assertEquals(text, termAtt.term());
assertEquals(text, termAtt.toString());
assertEquals(startOffset, offsetAtt.startOffset());
assertEquals(endOffset, offsetAtt.endOffset());
}
@ -478,7 +483,7 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
private static Token createToken(String term, int start, int offset)
{
Token token = new Token(start, offset);
token.setTermBuffer(term);
token.setEmpty().append(term);
return token;
}
@ -486,21 +491,15 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
public final static class TokenListStream extends TokenStream {
private Collection<Token> tokens;
TermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
PayloadAttribute payloadAtt;
OffsetAttribute offsetAtt;
TypeAttribute typeAtt;
FlagsAttribute flagsAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
public TokenListStream(Collection<Token> tokens) {
this.tokens = tokens;
termAtt = addAttribute(TermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
flagsAtt = addAttribute(FlagsAttribute.class);
}
private Iterator<Token> iterator;
@ -515,7 +514,7 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
}
Token prototype = iterator.next();
clearAttributes();
termAtt.setTermBuffer(prototype.termBuffer(), 0, prototype.termLength());
termAtt.copyBuffer(prototype.buffer(), 0, prototype.length());
posIncrAtt.setPositionIncrement(prototype.getPositionIncrement());
flagsAtt.setFlags(prototype.getFlags());
offsetAtt.setOffset(prototype.startOffset(), prototype.endOffset());

14
modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
public class TokenTypeSinkTokenizerTest extends BaseTokenStreamTestCase {
@ -41,11 +41,11 @@ public class TokenTypeSinkTokenizerTest extends BaseTokenStreamTestCase {
boolean seenDogs = false;
TermAttribute termAtt = ttf.addAttribute(TermAttribute.class);
CharTermAttribute termAtt = ttf.addAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = ttf.addAttribute(TypeAttribute.class);
ttf.reset();
while (ttf.incrementToken()) {
if (termAtt.term().equals("dogs")) {
if (termAtt.toString().equals("dogs")) {
seenDogs = true;
assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().equals("D") == true);
} else {
@ -64,20 +64,18 @@ public class TokenTypeSinkTokenizerTest extends BaseTokenStreamTestCase {
}
private class WordTokenFilter extends TokenFilter {
private TermAttribute termAtt;
private TypeAttribute typeAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private WordTokenFilter(TokenStream input) {
super(input);
termAtt = addAttribute(TermAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
}
@Override
public final boolean incrementToken() throws IOException {
if (!input.incrementToken()) return false;
if (termAtt.term().equals("dogs")) {
if (termAtt.toString().equals("dogs")) {
typeAtt.setType("D");
}
return true;

26
modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
View File

@ -22,11 +22,11 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Payload;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.Version;
@ -93,7 +93,7 @@ public class TestSnowball extends BaseTokenStreamTestCase {
public void testFilterTokens() throws Exception {
SnowballFilter filter = new SnowballFilter(new TestTokenStream(), "English");
TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = filter.getAttribute(OffsetAttribute.class);
TypeAttribute typeAtt = filter.getAttribute(TypeAttribute.class);
PayloadAttribute payloadAtt = filter.getAttribute(PayloadAttribute.class);
@ -102,7 +102,7 @@ public class TestSnowball extends BaseTokenStreamTestCase {
filter.incrementToken();
assertEquals("accent", termAtt.term());
assertEquals("accent", termAtt.toString());
assertEquals(2, offsetAtt.startOffset());
assertEquals(7, offsetAtt.endOffset());
assertEquals("wrd", typeAtt.type());
@ -112,27 +112,21 @@ public class TestSnowball extends BaseTokenStreamTestCase {
}
private final class TestTokenStream extends TokenStream {
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
private TypeAttribute typeAtt;
private PayloadAttribute payloadAtt;
private PositionIncrementAttribute posIncAtt;
private FlagsAttribute flagsAtt;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
TestTokenStream() {
super();
termAtt = addAttribute(TermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
typeAtt = addAttribute(TypeAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
posIncAtt = addAttribute(PositionIncrementAttribute.class);
flagsAtt = addAttribute(FlagsAttribute.class);
}
@Override
public boolean incrementToken() {
clearAttributes();
termAtt.setTermBuffer("accents");
termAtt.setEmpty().append("accents");
offsetAtt.setOffset(2, 7);
typeAtt.setType("wrd");
posIncAtt.setPositionIncrement(3);

2
modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilter.java
View File

@ -404,7 +404,7 @@ public class TestSynonymFilter extends BaseTokenStreamTestCase {
else {
clearAttributes();
Token token = tokens[index++];
termAtt.setEmpty().append(token.term());
termAtt.setEmpty().append(token);
offsetAtt.setOffset(token.startOffset(), token.endOffset());
posIncAtt.setPositionIncrement(token.getPositionIncrement());
flagsAtt.setFlags(token.getFlags());

560
modules/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
View File

@ -20,30 +20,20 @@ package org.apache.lucene.analysis.wikipedia;
import java.io.StringReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import static org.apache.lucene.analysis.wikipedia.WikipediaTokenizer.*;
/**
*
*
* Basic Tests for {@link WikipediaTokenizer}
**/
public class WikipediaTokenizerTest extends BaseTokenStreamTestCase {
protected static final String LINK_PHRASES = "click [[link here again]] click [http://lucene.apache.org here again] [[Category:a b c d]]";
public WikipediaTokenizerTest(String s) {
super(s);
}
public void testSimple() throws Exception {
String text = "This is a [[Category:foo]]";
WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(text));
@ -51,216 +41,85 @@ public class WikipediaTokenizerTest extends BaseTokenStreamTestCase {
new String[] { "This", "is", "a", "foo" },
new int[] { 0, 5, 8, 21 },
new int[] { 4, 7, 9, 24 },
new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", WikipediaTokenizer.CATEGORY },
new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", CATEGORY },
new int[] { 1, 1, 1, 1, },
text.length());
}
public void testHandwritten() throws Exception {
//make sure all tokens are in only one type
String test = "[[link]] This is a [[Category:foo]] Category This is a linked [[:Category:bar none withstanding]] " +
"Category This is (parens) This is a [[link]] This is an external URL [http://lucene.apache.org] " +
"Here is ''italics'' and ''more italics'', '''bold''' and '''''five quotes''''' " +
" This is a [[link|display info]] This is a period. Here is $3.25 and here is 3.50. Here's Johnny. " +
"==heading== ===sub head=== followed by some text [[Category:blah| ]] " +
"''[[Category:ital_cat]]'' here is some that is ''italics [[Category:foo]] but is never closed." +
"'''same [[Category:foo]] goes for this '''''and2 [[Category:foo]] and this" +
" [http://foo.boo.com/test/test/ Test Test] [http://foo.boo.com/test/test/test.html Test Test]" +
" [http://foo.boo.com/test/test/test.html?g=b&c=d Test Test] <ref>Citation</ref> <sup>martian</sup> <span class=\"glue\">code</span>";
Map<String,String> tcm = new HashMap<String,String>();//map tokens to types
tcm.put("link", WikipediaTokenizer.INTERNAL_LINK);
tcm.put("display", WikipediaTokenizer.INTERNAL_LINK);
tcm.put("info", WikipediaTokenizer.INTERNAL_LINK);
tcm.put("http://lucene.apache.org", WikipediaTokenizer.EXTERNAL_LINK_URL);
tcm.put("http://foo.boo.com/test/test/", WikipediaTokenizer.EXTERNAL_LINK_URL);
tcm.put("http://foo.boo.com/test/test/test.html", WikipediaTokenizer.EXTERNAL_LINK_URL);
tcm.put("http://foo.boo.com/test/test/test.html?g=b&c=d", WikipediaTokenizer.EXTERNAL_LINK_URL);
tcm.put("Test", WikipediaTokenizer.EXTERNAL_LINK);
// make sure all tokens are in only one type
String test = "[[link]] This is a [[Category:foo]] Category This is a linked [[:Category:bar none withstanding]] "
+ "Category This is (parens) This is a [[link]] This is an external URL [http://lucene.apache.org] "
+ "Here is ''italics'' and ''more italics'', '''bold''' and '''''five quotes''''' "
+ " This is a [[link|display info]] This is a period. Here is $3.25 and here is 3.50. Here's Johnny. "
+ "==heading== ===sub head=== followed by some text [[Category:blah| ]] "
+ "''[[Category:ital_cat]]'' here is some that is ''italics [[Category:foo]] but is never closed."
+ "'''same [[Category:foo]] goes for this '''''and2 [[Category:foo]] and this"
+ " [http://foo.boo.com/test/test/ Test Test] [http://foo.boo.com/test/test/test.html Test Test]"
+ " [http://foo.boo.com/test/test/test.html?g=b&c=d Test Test] <ref>Citation</ref> <sup>martian</sup> <span class=\"glue\">code</span>";
//alphanums
tcm.put("This", "<ALPHANUM>");
tcm.put("is", "<ALPHANUM>");
tcm.put("a", "<ALPHANUM>");
tcm.put("Category", "<ALPHANUM>");
tcm.put("linked", "<ALPHANUM>");
tcm.put("parens", "<ALPHANUM>");
tcm.put("external", "<ALPHANUM>");
tcm.put("URL", "<ALPHANUM>");
tcm.put("and", "<ALPHANUM>");
tcm.put("period", "<ALPHANUM>");
tcm.put("Here", "<ALPHANUM>");
tcm.put("Here's", "<APOSTROPHE>");
tcm.put("here", "<ALPHANUM>");
tcm.put("Johnny", "<ALPHANUM>");
tcm.put("followed", "<ALPHANUM>");
tcm.put("by", "<ALPHANUM>");
tcm.put("text", "<ALPHANUM>");
tcm.put("that", "<ALPHANUM>");
tcm.put("but", "<ALPHANUM>");
tcm.put("never", "<ALPHANUM>");
tcm.put("closed", "<ALPHANUM>");
tcm.put("goes", "<ALPHANUM>");
tcm.put("for", "<ALPHANUM>");
tcm.put("this", "<ALPHANUM>");
tcm.put("an", "<ALPHANUM>");
tcm.put("some", "<ALPHANUM>");
tcm.put("martian", "<ALPHANUM>");
tcm.put("code", "<ALPHANUM>");
tcm.put("foo", WikipediaTokenizer.CATEGORY);
tcm.put("bar", WikipediaTokenizer.CATEGORY);
tcm.put("none", WikipediaTokenizer.CATEGORY);
tcm.put("withstanding", WikipediaTokenizer.CATEGORY);
tcm.put("blah", WikipediaTokenizer.CATEGORY);
tcm.put("ital", WikipediaTokenizer.CATEGORY);
tcm.put("cat", WikipediaTokenizer.CATEGORY);
tcm.put("italics", WikipediaTokenizer.ITALICS);
tcm.put("more", WikipediaTokenizer.ITALICS);
tcm.put("bold", WikipediaTokenizer.BOLD);
tcm.put("same", WikipediaTokenizer.BOLD);
tcm.put("five", WikipediaTokenizer.BOLD_ITALICS);
tcm.put("and2", WikipediaTokenizer.BOLD_ITALICS);
tcm.put("quotes", WikipediaTokenizer.BOLD_ITALICS);
tcm.put("heading", WikipediaTokenizer.HEADING);
tcm.put("sub", WikipediaTokenizer.SUB_HEADING);
tcm.put("head", WikipediaTokenizer.SUB_HEADING);
tcm.put("Citation", WikipediaTokenizer.CITATION);
tcm.put("3.25", "<NUM>");
tcm.put("3.50", "<NUM>");
WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
int count = 0;
int numItalics = 0;
int numBoldItalics = 0;
int numCategory = 0;
int numCitation = 0;
TermAttribute termAtt = tf.addAttribute(TermAttribute.class);
TypeAttribute typeAtt = tf.addAttribute(TypeAttribute.class);
while (tf.incrementToken()) {
String tokText = termAtt.term();
//System.out.println("Text: " + tokText + " Type: " + token.type());
String expectedType = tcm.get(tokText);
assertTrue("expectedType is null and it shouldn't be for: " + tf.toString(), expectedType != null);
assertTrue(typeAtt.type() + " is not equal to " + expectedType + " for " + tf.toString(), typeAtt.type().equals(expectedType) == true);
count++;
if (typeAtt.type().equals(WikipediaTokenizer.ITALICS) == true){
numItalics++;
} else if (typeAtt.type().equals(WikipediaTokenizer.BOLD_ITALICS) == true){
numBoldItalics++;
} else if (typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true){
numCategory++;
}
else if (typeAtt.type().equals(WikipediaTokenizer.CITATION) == true){
numCitation++;
}
}
assertTrue("We have not seen enough tokens: " + count + " is not >= " + tcm.size(), count >= tcm.size());
assertTrue(numItalics + " does not equal: " + 4 + " for numItalics", numItalics == 4);
assertTrue(numBoldItalics + " does not equal: " + 3 + " for numBoldItalics", numBoldItalics == 3);
assertTrue(numCategory + " does not equal: " + 10 + " for numCategory", numCategory == 10);
assertTrue(numCitation + " does not equal: " + 1 + " for numCitation", numCitation == 1);
assertTokenStreamContents(tf,
new String[] {"link", "This", "is", "a",
"foo", "Category", "This", "is", "a", "linked", "bar", "none",
"withstanding", "Category", "This", "is", "parens", "This", "is", "a",
"link", "This", "is", "an", "external", "URL",
"http://lucene.apache.org", "Here", "is", "italics", "and", "more",
"italics", "bold", "and", "five", "quotes", "This", "is", "a", "link",
"display", "info", "This", "is", "a", "period", "Here", "is", "3.25",
"and", "here", "is", "3.50", "Here's", "Johnny", "heading", "sub",
"head", "followed", "by", "some", "text", "blah", "ital", "cat",
"here", "is", "some", "that", "is", "italics", "foo", "but", "is",
"never", "closed", "same", "foo", "goes", "for", "this", "and2", "foo",
"and", "this", "http://foo.boo.com/test/test/", "Test", "Test",
"http://foo.boo.com/test/test/test.html", "Test", "Test",
"http://foo.boo.com/test/test/test.html?g=b&c=d", "Test", "Test",
"Citation", "martian", "code"},
new String[] {INTERNAL_LINK,
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", CATEGORY, "<ALPHANUM>",
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", CATEGORY,
CATEGORY, CATEGORY, "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", INTERNAL_LINK,
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
EXTERNAL_LINK_URL, "<ALPHANUM>", "<ALPHANUM>", ITALICS, "<ALPHANUM>",
ITALICS, ITALICS, BOLD, "<ALPHANUM>", BOLD_ITALICS, BOLD_ITALICS,
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", INTERNAL_LINK, INTERNAL_LINK,
INTERNAL_LINK, "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
"<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<ALPHANUM>", "<ALPHANUM>",
"<ALPHANUM>", "<NUM>", "<APOSTROPHE>", "<ALPHANUM>", HEADING,
SUB_HEADING, SUB_HEADING, "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
"<ALPHANUM>", CATEGORY, CATEGORY, CATEGORY, "<ALPHANUM>", "<ALPHANUM>",
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", ITALICS, CATEGORY,
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", BOLD, CATEGORY,
"<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", BOLD_ITALICS, CATEGORY,
"<ALPHANUM>", "<ALPHANUM>", EXTERNAL_LINK_URL, EXTERNAL_LINK,
EXTERNAL_LINK, EXTERNAL_LINK_URL, EXTERNAL_LINK, EXTERNAL_LINK,
EXTERNAL_LINK_URL, EXTERNAL_LINK, EXTERNAL_LINK, CITATION,
"<ALPHANUM>", "<ALPHANUM>"});
}
public void testLinkPhrases() throws Exception {
WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(LINK_PHRASES));
checkLinkPhrases(tf);
}
private void checkLinkPhrases(WikipediaTokenizer tf) throws IOException {
TermAttribute termAtt = tf.addAttribute(TermAttribute.class);
PositionIncrementAttribute posIncrAtt = tf.addAttribute(PositionIncrementAttribute.class);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "click", termAtt.term().equals("click") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "link", termAtt.term().equals("link") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "here",
termAtt.term().equals("here") == true);
//The link, and here should be at the same position for phrases to work
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "again",
termAtt.term().equals("again") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "click",
termAtt.term().equals("click") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "http://lucene.apache.org",
termAtt.term().equals("http://lucene.apache.org") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "here",
termAtt.term().equals("here") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 0, posIncrAtt.getPositionIncrement() == 0);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "again",
termAtt.term().equals("again") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "a",
termAtt.term().equals("a") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "b",
termAtt.term().equals("b") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "c",
termAtt.term().equals("c") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "d",
termAtt.term().equals("d") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertFalse(tf.incrementToken());
assertTokenStreamContents(tf,
new String[] { "click", "link", "here", "again", "click",
"http://lucene.apache.org", "here", "again", "a", "b", "c", "d" },
new int[] { 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 });
}
public void testLinks() throws Exception {
String test = "[http://lucene.apache.org/java/docs/index.html#news here] [http://lucene.apache.org/java/docs/index.html?b=c here] [https://lucene.apache.org/java/docs/index.html?b=c here]";
WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
TermAttribute termAtt = tf.addAttribute(TermAttribute.class);
TypeAttribute typeAtt = tf.addAttribute(TypeAttribute.class);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html#news",
termAtt.term().equals("http://lucene.apache.org/java/docs/index.html#news") == true);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, typeAtt.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
tf.incrementToken();//skip here
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "http://lucene.apache.org/java/docs/index.html?b=c",
termAtt.term().equals("http://lucene.apache.org/java/docs/index.html?b=c") == true);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, typeAtt.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
tf.incrementToken();//skip here
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "https://lucene.apache.org/java/docs/index.html?b=c",
termAtt.term().equals("https://lucene.apache.org/java/docs/index.html?b=c") == true);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.EXTERNAL_LINK_URL, typeAtt.type().equals(WikipediaTokenizer.EXTERNAL_LINK_URL) == true);
assertTrue(tf.incrementToken());
assertFalse(tf.incrementToken());
assertTokenStreamContents(tf,
new String[] { "http://lucene.apache.org/java/docs/index.html#news", "here",
"http://lucene.apache.org/java/docs/index.html?b=c", "here",
"https://lucene.apache.org/java/docs/index.html?b=c", "here" },
new String[] { EXTERNAL_LINK_URL, EXTERNAL_LINK,
EXTERNAL_LINK_URL, EXTERNAL_LINK,
EXTERNAL_LINK_URL, EXTERNAL_LINK, });
}
public void testLucene1133() throws Exception {
@ -272,73 +131,13 @@ public class WikipediaTokenizerTest extends BaseTokenStreamTestCase {
checkLinkPhrases(tf);
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.UNTOKENIZED_ONLY, untoks);
TermAttribute termAtt = tf.addAttribute(TermAttribute.class);
PositionIncrementAttribute posIncrAtt = tf.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = tf.addAttribute(OffsetAttribute.class);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "a b c d",
termAtt.term().equals("a b c d") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 11, offsetAtt.startOffset() == 11);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 18, offsetAtt.endOffset() == 18);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "e f g",
termAtt.term().equals("e f g") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 32, offsetAtt.startOffset() == 32);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 37, offsetAtt.endOffset() == 37);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "link",
termAtt.term().equals("link") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 42, offsetAtt.startOffset() == 42);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 46, offsetAtt.endOffset() == 46);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "here",
termAtt.term().equals("here") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 47, offsetAtt.startOffset() == 47);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 51, offsetAtt.endOffset() == 51);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "link",
termAtt.term().equals("link") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 56, offsetAtt.startOffset() == 56);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 60, offsetAtt.endOffset() == 60);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "there",
termAtt.term().equals("there") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 61, offsetAtt.startOffset() == 61);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 66, offsetAtt.endOffset() == 66);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "italics here",
termAtt.term().equals("italics here") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 71, offsetAtt.startOffset() == 71);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 83, offsetAtt.endOffset() == 83);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "something",
termAtt.term().equals("something") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 86, offsetAtt.startOffset() == 86);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 95, offsetAtt.endOffset() == 95);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "more italics",
termAtt.term().equals("more italics") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 98, offsetAtt.startOffset() == 98);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 110, offsetAtt.endOffset() == 110);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "h i j",
termAtt.term().equals("h i j") == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 124, offsetAtt.startOffset() == 124);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 133, offsetAtt.endOffset() == 133);
assertFalse(tf.incrementToken());
assertTokenStreamContents(tf,
new String[] { "a b c d", "e f g", "link", "here", "link",
"there", "italics here", "something", "more italics", "h i j" },
new int[] { 11, 32, 42, 47, 56, 61, 71, 86, 98, 124 },
new int[] { 18, 37, 46, 51, 60, 66, 83, 95, 110, 133 },
new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }
);
}
public void testBoth() throws Exception {
@ -348,211 +147,26 @@ public class WikipediaTokenizerTest extends BaseTokenStreamTestCase {
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
//should output all the indivual tokens plus the untokenized tokens as well. Untokenized tokens
WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
TermAttribute termAtt = tf.addAttribute(TermAttribute.class);
TypeAttribute typeAtt = tf.addAttribute(TypeAttribute.class);
PositionIncrementAttribute posIncrAtt = tf.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = tf.addAttribute(OffsetAttribute.class);
assertTokenStreamContents(tf,
new String[] { "a b c d", "a", "b", "c", "d", "e f g", "e", "f", "g",
"link", "here", "link", "there", "italics here", "italics", "here",
"something", "more italics", "more", "italics", "h i j", "h", "i", "j" },
new int[] { 11, 11, 13, 15, 17, 32, 32, 34, 36, 42, 47, 56, 61, 71, 71, 79, 86, 98, 98, 103, 124, 124, 128, 132 },
new int[] { 18, 12, 14, 16, 18, 37, 33, 35, 37, 46, 51, 60, 66, 83, 78, 83, 95, 110, 102, 110, 133, 125, 129, 133 },
new int[] { 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1 }
);
// now check the flags, TODO: add way to check flags from BaseTokenStreamTestCase?
tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0,
0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
FlagsAttribute flagsAtt = tf.addAttribute(FlagsAttribute.class);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "a b c d",
termAtt.term().equals("a b c d") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(flagsAtt.getFlags() + " does not equal: " + WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG, flagsAtt.getFlags() == WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 11, offsetAtt.startOffset() == 11);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 18, offsetAtt.endOffset() == 18);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "a",
termAtt.term().equals("a") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 0, posIncrAtt.getPositionIncrement() == 0);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(flagsAtt.getFlags() + " equals: " + WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG + " and it shouldn't", flagsAtt.getFlags() != WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 11, offsetAtt.startOffset() == 11);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 12, offsetAtt.endOffset() == 12);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "b",
termAtt.term().equals("b") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 13, offsetAtt.startOffset() == 13);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 14, offsetAtt.endOffset() == 14);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "c",
termAtt.term().equals("c") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 15, offsetAtt.startOffset() == 15);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 16, offsetAtt.endOffset() == 16);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "d",
termAtt.term().equals("d") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 17, offsetAtt.startOffset() == 17);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 18, offsetAtt.endOffset() == 18);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "e f g",
termAtt.term().equals("e f g") == true);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(flagsAtt.getFlags() + " does not equal: " + WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG, flagsAtt.getFlags() == WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 32, offsetAtt.startOffset() == 32);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 37, offsetAtt.endOffset() == 37);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "e",
termAtt.term().equals("e") == true);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 0, posIncrAtt.getPositionIncrement() == 0);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 32, offsetAtt.startOffset() == 32);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 33, offsetAtt.endOffset() == 33);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "f",
termAtt.term().equals("f") == true);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 34, offsetAtt.startOffset() == 34);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 35, offsetAtt.endOffset() == 35);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "g",
termAtt.term().equals("g") == true);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 36, offsetAtt.startOffset() == 36);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 37, offsetAtt.endOffset() == 37);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "link",
termAtt.term().equals("link") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.INTERNAL_LINK, typeAtt.type().equals(WikipediaTokenizer.INTERNAL_LINK) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 42, offsetAtt.startOffset() == 42);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 46, offsetAtt.endOffset() == 46);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "here",
termAtt.term().equals("here") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.INTERNAL_LINK, typeAtt.type().equals(WikipediaTokenizer.INTERNAL_LINK) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 47, offsetAtt.startOffset() == 47);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 51, offsetAtt.endOffset() == 51);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "link",
termAtt.term().equals("link") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 56, offsetAtt.startOffset() == 56);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.INTERNAL_LINK, typeAtt.type().equals(WikipediaTokenizer.INTERNAL_LINK) == true);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 60, offsetAtt.endOffset() == 60);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "there",
termAtt.term().equals("there") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.INTERNAL_LINK, typeAtt.type().equals(WikipediaTokenizer.INTERNAL_LINK) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 61, offsetAtt.startOffset() == 61);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 66, offsetAtt.endOffset() == 66);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "italics here",
termAtt.term().equals("italics here") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.ITALICS, typeAtt.type().equals(WikipediaTokenizer.ITALICS) == true);
assertTrue(flagsAtt.getFlags() + " does not equal: " + WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG, flagsAtt.getFlags() == WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 71, offsetAtt.startOffset() == 71);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 83, offsetAtt.endOffset() == 83);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "italics",
termAtt.term().equals("italics") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 0, posIncrAtt.getPositionIncrement() == 0);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.ITALICS, typeAtt.type().equals(WikipediaTokenizer.ITALICS) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 71, offsetAtt.startOffset() == 71);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 78, offsetAtt.endOffset() == 78);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "here",
termAtt.term().equals("here") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.ITALICS, typeAtt.type().equals(WikipediaTokenizer.ITALICS) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 79, offsetAtt.startOffset() == 79);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 83, offsetAtt.endOffset() == 83);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "something",
termAtt.term().equals("something") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 86, offsetAtt.startOffset() == 86);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 95, offsetAtt.endOffset() == 95);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "more italics",
termAtt.term().equals("more italics") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.ITALICS, typeAtt.type().equals(WikipediaTokenizer.ITALICS) == true);
assertTrue(flagsAtt.getFlags() + " does not equal: " + WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG, flagsAtt.getFlags() == WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 98, offsetAtt.startOffset() == 98);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 110, offsetAtt.endOffset() == 110);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "more",
termAtt.term().equals("more") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 0, posIncrAtt.getPositionIncrement() == 0);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.ITALICS, typeAtt.type().equals(WikipediaTokenizer.ITALICS) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 98, offsetAtt.startOffset() == 98);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 102, offsetAtt.endOffset() == 102);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "italics",
termAtt.term().equals("italics") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.ITALICS, typeAtt.type().equals(WikipediaTokenizer.ITALICS) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 103, offsetAtt.startOffset() == 103);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 110, offsetAtt.endOffset() == 110);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "h i j",
termAtt.term().equals("h i j") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(flagsAtt.getFlags() + " does not equal: " + WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG, flagsAtt.getFlags() == WikipediaTokenizer.UNTOKENIZED_TOKEN_FLAG);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 124, offsetAtt.startOffset() == 124);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 133, offsetAtt.endOffset() == 133);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "h",
termAtt.term().equals("h") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 0, posIncrAtt.getPositionIncrement() == 0);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 124, offsetAtt.startOffset() == 124);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 125, offsetAtt.endOffset() == 125);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "i",
termAtt.term().equals("i") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 128, offsetAtt.startOffset() == 128);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 129, offsetAtt.endOffset() == 129);
assertTrue(tf.incrementToken());
assertTrue(termAtt.term() + " is not equal to " + "j",
termAtt.term().equals("j") == true);
assertTrue(posIncrAtt.getPositionIncrement() + " does not equal: " + 1, posIncrAtt.getPositionIncrement() == 1);
assertTrue(typeAtt.type() + " is not equal to " + WikipediaTokenizer.CATEGORY, typeAtt.type().equals(WikipediaTokenizer.CATEGORY) == true);
assertTrue(offsetAtt.startOffset() + " does not equal: " + 132, offsetAtt.startOffset() == 132);
assertTrue(offsetAtt.endOffset() + " does not equal: " + 133, offsetAtt.endOffset() == 133);
tf.reset();
for (int i = 0; i < expectedFlags.length; i++) {
assertTrue(tf.incrementToken());
assertEquals("flags " + i, expectedFlags[i], flagsAtt.getFlags());
}
assertFalse(tf.incrementToken());
tf.close();
}
}

Some files were not shown because too many files have changed in this diff Show More