mirror of https://github.com/apache/lucene.git
LUCENE-2384: Remove hack, as JFlex trunk now has the zzBuffer bug fixed
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@945130 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler
parent
acbf053b7c
commit
cd45643b96
|
|
@ -17,4 +17,5 @@
|
|||
|
||||
|
||||
WARNING: if you change StandardTokenizerImpl*.jflex and need to regenerate
|
||||
the tokenizer, only use the trunk version of JFlex 1.5 at the moment!
|
||||
the tokenizer, only use the trunk version of JFlex 1.5 (with a minimum
|
||||
SVN revision 591) at the moment!
|
||||
|
|
|
|||
|
|
@ -201,7 +201,7 @@ public final class StandardTokenizer extends Tokenizer {
|
|||
@Override
|
||||
public void reset(Reader reader) throws IOException {
|
||||
super.reset(reader);
|
||||
scanner.reset(reader);
|
||||
scanner.yyreset(reader);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */
|
||||
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */
|
||||
|
||||
package org.apache.lucene.analysis.standard;
|
||||
|
||||
|
|
@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
|||
/**
|
||||
* This class is a scanner generated by
|
||||
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
|
||||
* on 10.04.10 13:07 from the specification file
|
||||
* <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex</tt>
|
||||
* on 17.05.10 14:50 from the specification file
|
||||
* <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex</tt>
|
||||
*/
|
||||
class StandardTokenizerImpl31 implements StandardTokenizerInterface {
|
||||
|
||||
|
|
@ -379,17 +379,6 @@ public final void getText(CharTermAttribute t) {
|
|||
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the Tokenizer to a new Reader.
|
||||
*/
|
||||
public final void reset(Reader r) {
|
||||
// reset to default buffer size, if buffer has grown
|
||||
if (zzBuffer.length > ZZ_BUFFERSIZE) {
|
||||
zzBuffer = new char[ZZ_BUFFERSIZE];
|
||||
}
|
||||
yyreset(r);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -505,6 +494,8 @@ public final void reset(Reader r) {
|
|||
* <b>cannot</b> be reused (internal buffer is discarded and lost).
|
||||
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
|
||||
*
|
||||
* Internal scan buffer is resized down to its initial length, if it has grown.
|
||||
*
|
||||
* @param reader the new input stream
|
||||
*/
|
||||
public final void yyreset(java.io.Reader reader) {
|
||||
|
|
@ -516,6 +507,8 @@ public final void reset(Reader r) {
|
|||
zzCurrentPos = zzMarkedPos = 0;
|
||||
yyline = yychar = yycolumn = 0;
|
||||
zzLexicalState = YYINITIAL;
|
||||
if (zzBuffer.length > ZZ_BUFFERSIZE)
|
||||
zzBuffer = new char[ZZ_BUFFERSIZE];
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -67,17 +67,6 @@ public final void getText(CharTermAttribute t) {
|
|||
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the Tokenizer to a new Reader.
|
||||
*/
|
||||
public final void reset(Reader r) {
|
||||
// reset to default buffer size, if buffer has grown
|
||||
if (zzBuffer.length > ZZ_BUFFERSIZE) {
|
||||
zzBuffer = new char[ZZ_BUFFERSIZE];
|
||||
}
|
||||
yyreset(r);
|
||||
}
|
||||
|
||||
%}
|
||||
|
||||
THAI = [\u0E00-\u0E59]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */
|
||||
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */
|
||||
|
||||
package org.apache.lucene.analysis.standard;
|
||||
|
||||
|
|
@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
|
|||
/**
|
||||
* This class is a scanner generated by
|
||||
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
|
||||
* on 10.04.10 13:07 from the specification file
|
||||
* <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex</tt>
|
||||
* on 17.05.10 14:50 from the specification file
|
||||
* <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex</tt>
|
||||
*/
|
||||
class StandardTokenizerImplOrig implements StandardTokenizerInterface {
|
||||
|
||||
|
|
@ -375,17 +375,6 @@ public final void getText(CharTermAttribute t) {
|
|||
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the Tokenizer to a new Reader.
|
||||
*/
|
||||
public final void reset(Reader r) {
|
||||
// reset to default buffer size, if buffer has grown
|
||||
if (zzBuffer.length > ZZ_BUFFERSIZE) {
|
||||
zzBuffer = new char[ZZ_BUFFERSIZE];
|
||||
}
|
||||
yyreset(r);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -501,6 +490,8 @@ public final void reset(Reader r) {
|
|||
* <b>cannot</b> be reused (internal buffer is discarded and lost).
|
||||
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
|
||||
*
|
||||
* Internal scan buffer is resized down to its initial length, if it has grown.
|
||||
*
|
||||
* @param reader the new input stream
|
||||
*/
|
||||
public final void yyreset(java.io.Reader reader) {
|
||||
|
|
@ -512,6 +503,8 @@ public final void reset(Reader r) {
|
|||
zzCurrentPos = zzMarkedPos = 0;
|
||||
yyline = yychar = yycolumn = 0;
|
||||
zzLexicalState = YYINITIAL;
|
||||
if (zzBuffer.length > ZZ_BUFFERSIZE)
|
||||
zzBuffer = new char[ZZ_BUFFERSIZE];
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -67,17 +67,6 @@ public final void getText(CharTermAttribute t) {
|
|||
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the Tokenizer to a new Reader.
|
||||
*/
|
||||
public final void reset(Reader r) {
|
||||
// reset to default buffer size, if buffer has grown
|
||||
if (zzBuffer.length > ZZ_BUFFERSIZE) {
|
||||
zzBuffer = new char[ZZ_BUFFERSIZE];
|
||||
}
|
||||
yyreset(r);
|
||||
}
|
||||
|
||||
%}
|
||||
|
||||
THAI = [\u0E00-\u0E59]
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ interface StandardTokenizerInterface {
|
|||
*
|
||||
* @param reader the new input stream
|
||||
*/
|
||||
void reset(Reader reader);
|
||||
void yyreset(Reader reader);
|
||||
|
||||
/**
|
||||
* Returns the length of the matched text region.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 12:56 */
|
||||
/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:51 */
|
||||
|
||||
package org.apache.lucene.analysis.wikipedia;
|
||||
|
||||
|
|
@ -25,8 +25,8 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute;
|
|||
/**
|
||||
* This class is a scanner generated by
|
||||
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
|
||||
* on 10.04.10 12:56 from the specification file
|
||||
* <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
|
||||
* on 17.05.10 14:51 from the specification file
|
||||
* <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
|
||||
*/
|
||||
class WikipediaTokenizerImpl {
|
||||
|
||||
|
|
@ -613,6 +613,8 @@ final int setText(StringBuilder buffer){
|
|||
* <b>cannot</b> be reused (internal buffer is discarded and lost).
|
||||
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
|
||||
*
|
||||
* Internal scan buffer is resized down to its initial length, if it has grown.
|
||||
*
|
||||
* @param reader the new input stream
|
||||
*/
|
||||
public final void yyreset(java.io.Reader reader) {
|
||||
|
|
@ -624,6 +626,8 @@ final int setText(StringBuilder buffer){
|
|||
zzCurrentPos = zzMarkedPos = 0;
|
||||
yyline = yychar = yycolumn = 0;
|
||||
zzLexicalState = YYINITIAL;
|
||||
if (zzBuffer.length > ZZ_BUFFERSIZE)
|
||||
zzBuffer = new char[ZZ_BUFFERSIZE];
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue