SOLR-1662 -- Added Javadocs in BufferedTokenStream and fixed incorrect cloning in TestBufferedTokenStream

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@891889 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -169,6 +169,9 @@ Other Changes
 
 * SOLR-1637: Remove ALIAS command
 
+* SOLR-1662: Added Javadocs in BufferedTokenStream and fixed incorrect cloning
+  in TestBufferedTokenStream (Robert Muir, Uwe Schindler via shalin)
+
 Build
 ----------------------
 

src/java/org/apache/solr/analysis/BufferedTokenStream.java

@@ -46,12 +46,15 @@ import java.util.LinkedList;
  *   public MyTokenStream(TokenStream input) {super(input);}
  *   protected Token process(Token t) throws IOException {
  *     if ("A".equals(t.termText()) && "B".equals(peek(1).termText()))
- *       write(t);
+ *       write((Token)t.clone());
  *     return t;
  *   }
  * }
  * </pre>
  *
+ * NOTE: BufferedTokenStream does not clone() any Tokens. This is instead the 
+ * responsibility of the implementing subclass. In the "A" "B" => "A" "A" "B"
+ * example above, the subclass must clone the additional "A" it creates.
  * 
  * @version $Id$
  */

src/test/org/apache/solr/analysis/TestBufferedTokenStream.java

@@ -50,7 +50,7 @@ public class TestBufferedTokenStream extends BaseTokenTestCase {
     protected Token process(Token t) throws IOException {
       if ("A".equals(new String(t.termBuffer(), 0, t.termLength())) && 
           "B".equals(new String(peek(1).termBuffer(), 0, peek(1).termLength())))
-        write(t);
+        write((Token)t.clone());
       return t;
     }
   }