LUCENE-2384: Remove hack, as JFlex trunk now has the zzBuffer bug fixed

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@945130 13f79535-47bb-0310-9956-ffa450edef68

lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt

@@ -17,4 +17,5 @@
 
 
 WARNING: if you change StandardTokenizerImpl*.jflex and need to regenerate
-      the tokenizer, only use the trunk version of JFlex 1.5 at the moment!
+      the tokenizer, only use the trunk version of JFlex 1.5 (with a minimum
+      SVN revision 591) at the moment!

lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java

@@ -201,7 +201,7 @@ public final class StandardTokenizer extends Tokenizer {
   @Override
   public void reset(Reader reader) throws IOException {
     super.reset(reader);
-    scanner.reset(reader);
+    scanner.yyreset(reader);
   }
 
   /**

lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java

@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */
 
 package org.apache.lucene.analysis.standard;
 
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 10.04.10 13:07 from the specification file
+ * on 17.05.10 14:50 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex</tt>
+ * <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex</tt>
  */
 class StandardTokenizerImpl31 implements StandardTokenizerInterface {
 
@@ -379,17 +379,6 @@ public final void getText(CharTermAttribute t) {
   t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 }
 
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
-  // reset to default buffer size, if buffer has grown
-  if (zzBuffer.length > ZZ_BUFFERSIZE) {
-    zzBuffer = new char[ZZ_BUFFERSIZE];
-  }
-  yyreset(r);
-}
-
 
 
   /**
@@ -505,6 +494,8 @@ public final void reset(Reader r) {
    * <b>cannot</b> be reused (internal buffer is discarded and lost).
    * Lexical state is set to <tt>ZZ_INITIAL</tt>.
    *
+   * Internal scan buffer is resized down to its initial length, if it has grown.
+   *
    * @param reader   the new input stream 
    */
   public final void yyreset(java.io.Reader reader) {
@@ -516,6 +507,8 @@ public final void reset(Reader r) {
     zzCurrentPos = zzMarkedPos = 0;
     yyline = yychar = yycolumn = 0;
     zzLexicalState = YYINITIAL;
+    if (zzBuffer.length > ZZ_BUFFERSIZE)
+      zzBuffer = new char[ZZ_BUFFERSIZE];
   }
 
 

lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex

@@ -67,17 +67,6 @@ public final void getText(CharTermAttribute t) {
   t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 }
 
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
-  // reset to default buffer size, if buffer has grown
-  if (zzBuffer.length > ZZ_BUFFERSIZE) {
-    zzBuffer = new char[ZZ_BUFFERSIZE];
-  }
-  yyreset(r);
-}
-
 %}
 
 THAI       = [\u0E00-\u0E59]

lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java

@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */
 
 package org.apache.lucene.analysis.standard;
 
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 10.04.10 13:07 from the specification file
+ * on 17.05.10 14:50 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex</tt>
+ * <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex</tt>
  */
 class StandardTokenizerImplOrig implements StandardTokenizerInterface {
 
@@ -375,17 +375,6 @@ public final void getText(CharTermAttribute t) {
   t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 }
 
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
-  // reset to default buffer size, if buffer has grown
-  if (zzBuffer.length > ZZ_BUFFERSIZE) {
-    zzBuffer = new char[ZZ_BUFFERSIZE];
-  }
-  yyreset(r);
-}
-
 
 
   /**
@@ -501,6 +490,8 @@ public final void reset(Reader r) {
    * <b>cannot</b> be reused (internal buffer is discarded and lost).
    * Lexical state is set to <tt>ZZ_INITIAL</tt>.
    *
+   * Internal scan buffer is resized down to its initial length, if it has grown.
+   *
    * @param reader   the new input stream 
    */
   public final void yyreset(java.io.Reader reader) {
@@ -512,6 +503,8 @@ public final void reset(Reader r) {
     zzCurrentPos = zzMarkedPos = 0;
     yyline = yychar = yycolumn = 0;
     zzLexicalState = YYINITIAL;
+    if (zzBuffer.length > ZZ_BUFFERSIZE)
+      zzBuffer = new char[ZZ_BUFFERSIZE];
   }
 
 

lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex

@@ -67,17 +67,6 @@ public final void getText(CharTermAttribute t) {
   t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 }
 
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
-  // reset to default buffer size, if buffer has grown
-  if (zzBuffer.length > ZZ_BUFFERSIZE) {
-    zzBuffer = new char[ZZ_BUFFERSIZE];
-  }
-  yyreset(r);
-}
-
 %}
 
 THAI       = [\u0E00-\u0E59]

lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java

@@ -47,7 +47,7 @@ interface StandardTokenizerInterface {
    *
    * @param reader   the new input stream 
    */
-  void reset(Reader reader);
+  void yyreset(Reader reader);
 
   /**
    * Returns the length of the matched text region.

modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java

@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 12:56 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:51 */
 
 package org.apache.lucene.analysis.wikipedia;
 
@@ -25,8 +25,8 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 10.04.10 12:56 from the specification file
+ * on 17.05.10 14:51 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
+ * <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
  */
 class WikipediaTokenizerImpl {
 
@@ -613,6 +613,8 @@ final int setText(StringBuilder buffer){
    * <b>cannot</b> be reused (internal buffer is discarded and lost).
    * Lexical state is set to <tt>ZZ_INITIAL</tt>.
    *
+   * Internal scan buffer is resized down to its initial length, if it has grown.
+   *
    * @param reader   the new input stream 
    */
   public final void yyreset(java.io.Reader reader) {
@@ -624,6 +626,8 @@ final int setText(StringBuilder buffer){
     zzCurrentPos = zzMarkedPos = 0;
     yyline = yychar = yycolumn = 0;
     zzLexicalState = YYINITIAL;
+    if (zzBuffer.length > ZZ_BUFFERSIZE)
+      zzBuffer = new char[ZZ_BUFFERSIZE];
   }