LUCENE-1760: TokenStream API javadoc improvements

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@797714 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/lucene/analysis/TeeSinkTokenFilter.java

@@ -150,9 +150,13 @@ public final class TeeSinkTokenFilter extends TokenFilter {
   }
   
   /**
-   * TODO: Missing Docs
+   * A filter that decides which {@link AttributeSource} states to store in the sink.
    */
   public static interface SinkFilter {
+    /**
+     * Returns true, iff the current state of the passed-in {@link AttributeSource} shall be stored
+     * in the sink. 
+     */
     boolean accept(AttributeSource source);
   }
   

src/java/org/apache/lucene/analysis/Token.java

@@ -235,7 +235,7 @@ public class Token extends AttributeImpl
    *  @param text term text
    *  @param start start offset
    *  @param end end offset
-   *  @deprecated
+   *  @deprecated Use {@link #Token(char[], int, int, int, int)} instead.
    */
   public Token(String text, int start, int end) {
     termText = text;
@@ -251,7 +251,7 @@ public class Token extends AttributeImpl
    *  @param start start offset
    *  @param end end offset
    *  @param typ token type
-   *  @deprecated
+   *  @deprecated Use {@link #Token(char[], int, int, int, int)} and {@link #setType(String)} instead.
    */
   public Token(String text, int start, int end, String typ) {
     termText = text;
@@ -269,7 +269,7 @@ public class Token extends AttributeImpl
    * @param start
    * @param end
    * @param flags token type bits
-   * @deprecated
+   * @deprecated Use {@link #Token(char[], int, int, int, int)} and {@link #setFlags(int)} instead.
    */
   public Token(String text, int start, int end, int flags) {
     termText = text;

src/java/org/apache/lucene/analysis/TokenFilter.java

@@ -26,10 +26,6 @@ import java.io.IOException;
   {@link #incrementToken()} if the new TokenStream API is used
   and {@link #next(Token)} or {@link #next()} if the old
   TokenStream API is used.
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
   <p>
   See {@link TokenStream}
   */

src/java/org/apache/lucene/analysis/TokenStream.java

@@ -68,11 +68,7 @@ import org.apache.lucene.util.AttributeSource;
   {@link TeeSinkTokenFilter}). For this usecase
   {@link AttributeSource#captureState} and {@link AttributeSource#restoreState} can be used.  
  * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
+ */
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
-  */
-
 public abstract class TokenStream extends AttributeSource {
 
   /** @deprecated Remove this when old API is removed! */

src/java/org/apache/lucene/analysis/Tokenizer.java

@@ -30,10 +30,6 @@ import java.io.IOException;
   <p>
   NOTE: subclasses overriding {@link #next(Token)} must  
   call {@link Token#clear()}.
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 
 public abstract class Tokenizer extends TokenStream {

src/java/org/apache/lucene/analysis/package.html

@@ -466,7 +466,7 @@ This should be the usual behavior. However, there is also an expert-API that all
 and returns an actual instance. You can implement your own factory if you need to change the default behavior. <br/><br/>
 
 Now here is the actual class that implements our new Attribute. Notice that the class has to extend
-{@link org.apache.lucene.util.AttributeSource.AttributeImpl}:
+{@link org.apache.lucene.util.AttributeImpl}:
 
 <pre>
 public final class PartOfSpeechAttributeImpl extends AttributeImpl 

src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java

@@ -22,12 +22,6 @@ import org.apache.lucene.util.Attribute;
 /**
  * This attribute can be used to pass different flags down the tokenizer chain,
  * e. g. from one TokenFilter to another one. 
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
-
  */
 public interface FlagsAttribute extends Attribute {
   /**

src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java

@@ -24,12 +24,6 @@ import org.apache.lucene.util.AttributeImpl;
 /**
  * This attribute can be used to pass different flags down the tokenizer chain,
  * e. g. from one TokenFilter to another one. 
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
-
  */
 public class FlagsAttributeImpl extends AttributeImpl implements FlagsAttribute, Cloneable, Serializable {
   private int flags = 0;

src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java

@@ -21,11 +21,6 @@ import org.apache.lucene.util.Attribute;
 
 /**
  * The start and end character offset of a Token. 
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public interface OffsetAttribute extends Attribute {
   /** Returns this Token's starting offset, the position of the first character

src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java

@@ -23,11 +23,6 @@ import org.apache.lucene.util.AttributeImpl;
 
 /**
  * The start and end character offset of a Token. 
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public class OffsetAttributeImpl extends AttributeImpl implements OffsetAttribute, Cloneable, Serializable {
   private int startOffset;

src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java

@@ -22,11 +22,6 @@ import org.apache.lucene.util.Attribute;
 
 /**
  * The payload of a Token. See also {@link Payload}.
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public interface PayloadAttribute extends Attribute {
   /**

src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java

@@ -24,11 +24,6 @@ import org.apache.lucene.util.AttributeImpl;
 
 /**
  * The payload of a Token. See also {@link Payload}.
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttribute, Cloneable, Serializable {
   private Payload payload;  

src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java

@@ -43,11 +43,6 @@ import org.apache.lucene.util.Attribute;
  *
  * </ul>
  * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
- * 
  * @see org.apache.lucene.index.TermPositions
  */
 public interface PositionIncrementAttribute extends Attribute {

src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java

@@ -45,13 +45,6 @@ import org.apache.lucene.util.AttributeImpl;
  * occur with no intervening stop words.
  *
  * </ul>
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
- * 
- * @see org.apache.lucene.index.TermPositions
  */
 public class PositionIncrementAttributeImpl extends AttributeImpl implements PositionIncrementAttribute, Cloneable, Serializable {
   private int positionIncrement = 1;

src/java/org/apache/lucene/analysis/tokenattributes/TermAttribute.java

@@ -21,11 +21,6 @@ import org.apache.lucene.util.Attribute;
 
 /**
  * The term text of a Token.
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public interface TermAttribute extends Attribute {
   /** Returns the Token's term text.

src/java/org/apache/lucene/analysis/tokenattributes/TermAttributeImpl.java

@@ -24,11 +24,6 @@ import org.apache.lucene.util.AttributeImpl;
 
 /**
  * The term text of a Token.
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public class TermAttributeImpl extends AttributeImpl implements TermAttribute, Cloneable, Serializable {
   private static int MIN_BUFFER_SIZE = 10;

src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java

@@ -21,11 +21,6 @@ import org.apache.lucene.util.Attribute;
 
 /**
  * A Token's lexical type. The Default value is "word". 
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public interface TypeAttribute extends Attribute {
   /** Returns this Token's lexical type.  Defaults to "word". */

src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java

@@ -23,11 +23,6 @@ import org.apache.lucene.util.AttributeImpl;
 
 /**
  * A Token's lexical type. The Default value is "word". 
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public class TypeAttributeImpl extends AttributeImpl implements TypeAttribute, Cloneable, Serializable {
   private String type;

src/java/org/apache/lucene/util/Attribute.java

@@ -19,11 +19,6 @@ package org.apache.lucene.util;
 
 /**
  * Base interface for attributes.
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public interface Attribute {
   public void clear();

src/java/org/apache/lucene/util/AttributeImpl.java

@@ -26,10 +26,6 @@ import java.lang.reflect.Field;
  * <p>
  * Attributes are used to add data in a dynamic, yet type-safe way to a source
  * of usually streamed objects, e. g. a {@link org.apache.lucene.analysis.TokenStream}.
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public abstract class AttributeImpl implements Cloneable, Serializable {  
   /**

src/java/org/apache/lucene/util/AttributeSource.java

@@ -35,11 +35,6 @@ import org.apache.lucene.analysis.TokenStream; // for javadocs
  * the {@link #addAttribute(Class)}, which then checks if an instance of
  * that type is already present. If yes, it returns the instance, otherwise
  * it creates a new instance and returns it.
- * 
- * <p><font color="#FF0000">
- * WARNING: The status of the new TokenStream, AttributeSource and Attributes is experimental. 
- * The APIs introduced in these classes with Lucene 2.9 might change in the future. 
- * We will make our best efforts to keep the APIs backwards-compatible.</font>
  */
 public class AttributeSource {
   /**