Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added javadocs about UNTOKENIZED_TOKEN_FLAG conundrum 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@614905 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Grant Ingersoll 2008-01-24 15:42:50 +00:00
parent 305c47f500
commit 7f09c9f00e
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
View File

@ -49,7 +49,9 @@ public class WikipediaTokenizer extends Tokenizer {
public static final int TOKENS_ONLY = 0;
public static final int UNTOKENIZED_ONLY = 1;
public static final int BOTH = 2;
/**
* This flag is used to indicate that the produced "Token" would, if {@link #TOKENS_ONLY} was used, produce multiple tokens.
*/
public static final int UNTOKENIZED_TOKEN_FLAG = 1;
/**
* A private instance of the JFlex-constructed scanner