From 7f09c9f00e759f2477a2a19d845379c898f3c22b Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Thu, 24 Jan 2008 15:42:50 +0000 Subject: [PATCH] Added javadocs about UNTOKENIZED_TOKEN_FLAG conundrum git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@614905 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/wikipedia/analysis/WikipediaTokenizer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java b/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java index 3dc685dc7e8..d1b3997c6c9 100644 --- a/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java +++ b/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java @@ -49,7 +49,9 @@ public class WikipediaTokenizer extends Tokenizer { public static final int TOKENS_ONLY = 0; public static final int UNTOKENIZED_ONLY = 1; public static final int BOTH = 2; - + /** + * This flag is used to indicate that the produced "Token" would, if {@link #TOKENS_ONLY} was used, produce multiple tokens. + */ public static final int UNTOKENIZED_TOKEN_FLAG = 1; /** * A private instance of the JFlex-constructed scanner