Note that - * this does not do line-number counting, but instead keeps track of the - * character position of the token in the input, as required by Lucene's {@link - * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API. */ +/** + * An efficient implementation of JavaCC's CharStream interface. + *
+ * Note that this does not do line-number counting, but instead keeps track of the + * character position of the token in the input, as required by Lucene's + * {@link org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API. + */ public final class FastCharStream implements CharStream { + // See SOLR-11314 + private final static IOException READ_PAST_EOF = new IOException("Read past EOF."); + char[] buffer = null; int bufferLength = 0; // end of valid chars @@ -33,7 +39,9 @@ public final class FastCharStream implements CharStream { Reader input; // source of chars - /** Constructs from a Reader. */ + /** + * Constructs from a Reader. + */ public FastCharStream(Reader r) { input = r; } @@ -45,7 +53,7 @@ public final class FastCharStream implements CharStream { return buffer[bufferPosition++]; } - private final void refill() throws IOException { + private void refill() throws IOException { int newPosition = bufferLength - tokenStart; if (tokenStart == 0) { // token won't fit in buffer @@ -66,9 +74,9 @@ public final class FastCharStream implements CharStream { tokenStart = 0; int charsRead = // fill space in buffer - input.read(buffer, newPosition, buffer.length-newPosition); + input.read(buffer, newPosition, buffer.length - newPosition); if (charsRead == -1) - throw new IOException("read past eof"); + throw READ_PAST_EOF; else bufferLength += charsRead; } @@ -101,60 +109,27 @@ public final class FastCharStream implements CharStream { try { input.close(); } catch (IOException e) { + throw new UncheckedIOException(e); } } - /** - * Returns the column position of the character last read. - * @deprecated see: #getEndColumn - */ - @Deprecated - @Override - public final int getColumn() { - return bufferStart + bufferPosition; - } - /** - * @deprecated see: #getEndLine - */ - @Deprecated - @Override - public final int getLine() { - return 1; - } @Override public final int getEndColumn() { return bufferStart + bufferPosition; } + @Override public final int getEndLine() { return 1; } + @Override public final int getBeginColumn() { return bufferStart + tokenStart; } + @Override public final int getBeginLine() { return 1; } - - @Override - public void setTabSize(int i) { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public int getTabSize() { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public boolean getTrackLineColumn() { - return false; - } - - @Override - public void setTrackLineColumn(boolean trackLineColumn) { - throw new RuntimeException("Line/Column tracking not implemented."); - } } diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/package-info.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/package-info.java new file mode 100644 index 00000000000..dc30e767015 --- /dev/null +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/package-info.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * This package contains reusable parts for javacc-generated + * grammars (query parsers). + * + * @see org.apache.lucene.queryparser.charstream.CharStream + * @see org.apache.lucene.queryparser.charstream.FastCharStream + */ +package org.apache.lucene.queryparser.charstream; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java deleted file mode 100644 index e01b90a60ea..00000000000 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.queryparser.classic; - -import java.io.*; - -/** An efficient implementation of JavaCC's CharStream interface.
Note that - * this does not do line-number counting, but instead keeps track of the - * character position of the token in the input, as required by Lucene's {@link - * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API. - * */ -public final class FastCharStream implements CharStream { - char[] buffer = null; - - int bufferLength = 0; // end of valid chars - int bufferPosition = 0; // next char to read - - int tokenStart = 0; // offset in buffer - int bufferStart = 0; // position in file of buffer - - Reader input; // source of chars - - /** Constructs from a Reader. */ - public FastCharStream(Reader r) { - input = r; - } - - @Override - public final char readChar() throws IOException { - if (bufferPosition >= bufferLength) - refill(); - return buffer[bufferPosition++]; - } - - private final void refill() throws IOException { - int newPosition = bufferLength - tokenStart; - - if (tokenStart == 0) { // token won't fit in buffer - if (buffer == null) { // first time: alloc buffer - buffer = new char[2048]; - } else if (bufferLength == buffer.length) { // grow buffer - char[] newBuffer = new char[buffer.length*2]; - System.arraycopy(buffer, 0, newBuffer, 0, bufferLength); - buffer = newBuffer; - } - } else { // shift token to front - System.arraycopy(buffer, tokenStart, buffer, 0, newPosition); - } - - bufferLength = newPosition; // update state - bufferPosition = newPosition; - bufferStart += tokenStart; - tokenStart = 0; - - int charsRead = // fill space in buffer - input.read(buffer, newPosition, buffer.length-newPosition); - if (charsRead == -1) - throw new IOException("read past eof"); - else - bufferLength += charsRead; - } - - @Override - public final char BeginToken() throws IOException { - tokenStart = bufferPosition; - return readChar(); - } - - @Override - public final void backup(int amount) { - bufferPosition -= amount; - } - - @Override - public final String GetImage() { - return new String(buffer, tokenStart, bufferPosition - tokenStart); - } - - @Override - public final char[] GetSuffix(int len) { - char[] value = new char[len]; - System.arraycopy(buffer, bufferPosition - len, value, 0, len); - return value; - } - - @Override - public final void Done() { - try { - input.close(); - } catch (IOException e) { - } - } - - /** - * Returns the column position of the character last read. - * @deprecated see: #getEndColumn - */ - @Deprecated - @Override - public final int getColumn() { - return bufferStart + bufferPosition; - } - /** - * @deprecated see #getEndLine - */ - @Override - @Deprecated - public final int getLine() { - return 1; - } - @Override - public final int getEndColumn() { - return bufferStart + bufferPosition; - } - @Override - public final int getEndLine() { - return 1; - } - @Override - public final int getBeginColumn() { - return bufferStart + tokenStart; - } - @Override - public final int getBeginLine() { - return 1; - } - - @Override - public void setTabSize(int i) { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public int getTabSize() { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public boolean getTrackLineColumn() { - return false; - } - - @Override - public void setTrackLineColumn(boolean trackLineColumn) { - throw new RuntimeException("Line/Column tracking not implemented."); - } -} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java index f7da7e01889..27488161260 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java @@ -15,6 +15,8 @@ import org.apache.lucene.document.DateTools; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; /** * This class is generated by JavaCC. The most important method is @@ -706,22 +708,6 @@ if (splitOnWhitespace == false) { finally { jj_save(2, xla); } } - private boolean jj_3R_3() - { - if (jj_scan_token(TERM)) return true; - jj_lookingAhead = true; - jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind); - jj_lookingAhead = false; - if (!jj_semLA || jj_3R_6()) return true; - Token xsp; - if (jj_3R_7()) return true; - while (true) { - xsp = jj_scanpos; - if (jj_3R_7()) { jj_scanpos = xsp; break; } - } - return false; - } - private boolean jj_3R_6() { return false; @@ -770,6 +756,22 @@ if (splitOnWhitespace == false) { return false; } + private boolean jj_3R_3() + { + if (jj_scan_token(TERM)) return true; + jj_lookingAhead = true; + jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind); + jj_lookingAhead = false; + if (!jj_semLA || jj_3R_6()) return true; + Token xsp; + if (jj_3R_7()) return true; + while (true) { + xsp = jj_scanpos; + if (jj_3R_7()) { jj_scanpos = xsp; break; } + } + return false; + } + /** Generated Token Manager. */ public QueryParserTokenManager token_source; /** Current token. */ diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj index fdc0cd0ed5e..11ba876b332 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj @@ -38,6 +38,8 @@ import org.apache.lucene.document.DateTools; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; /** * This class is generated by JavaCC. The most important method is diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java index a1f8fa582ad..6afe913d0b1 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java @@ -25,6 +25,8 @@ import java.util.regex.Pattern; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.DateTools; import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; import org.apache.lucene.queryparser.classic.QueryParser.Operator; import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration; import org.apache.lucene.search.*; diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java index db84cc7d006..10d09505d25 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java @@ -1,6 +1,9 @@ /* QueryParserTokenManager.java */ /* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */ package org.apache.lucene.queryparser.classic; + import org.apache.lucene.queryparser.charstream.CharStream; + + diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java deleted file mode 100644 index 575be8e49aa..00000000000 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java +++ /dev/null @@ -1,120 +0,0 @@ -/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 7.0 */ -/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ -package org.apache.lucene.queryparser.flexible.standard.parser; - -/** - * This interface describes a character stream that maintains line and - * column number positions of the characters. It also has the capability - * to backup the stream to some extent. An implementation of this - * interface is used in the TokenManager implementation generated by - * JavaCCParser. - * - * All the methods except backup can be implemented in any fashion. backup - * needs to be implemented correctly for the correct operation of the lexer. - * Rest of the methods are all used to get information like line number, - * column number and the String that constitutes a token and are not used - * by the lexer. Hence their implementation won't affect the generated lexer's - * operation. - */ - -public -interface CharStream { - - /** - * Returns the next character from the selected input. The method - * of selecting the input is the responsibility of the class - * implementing this interface. Can throw any java.io.IOException. - */ - char readChar() throws java.io.IOException; - - @Deprecated - /** - * Returns the column position of the character last read. - * @deprecated - * @see #getEndColumn - */ - int getColumn(); - - @Deprecated - /** - * Returns the line number of the character last read. - * @deprecated - * @see #getEndLine - */ - int getLine(); - - /** - * Returns the column number of the last character for current token (being - * matched after the last call to BeginTOken). - */ - int getEndColumn(); - - /** - * Returns the line number of the last character for current token (being - * matched after the last call to BeginTOken). - */ - int getEndLine(); - - /** - * Returns the column number of the first character for current token (being - * matched after the last call to BeginTOken). - */ - int getBeginColumn(); - - /** - * Returns the line number of the first character for current token (being - * matched after the last call to BeginTOken). - */ - int getBeginLine(); - - /** - * Backs up the input stream by amount steps. Lexer calls this method if it - * had already read some characters, but could not use them to match a - * (longer) token. So, they will be used again as the prefix of the next - * token and it is the implementation's responsibility to do this right. - */ - void backup(int amount); - - /** - * Returns the next character that marks the beginning of the next token. - * All characters must remain in the buffer between two successive calls - * to this method to implement backup correctly. - */ - char BeginToken() throws java.io.IOException; - - /** - * Returns a string made up of characters from the marked token beginning - * to the current buffer position. Implementations have the choice of returning - * anything that they want to. For example, for efficiency, one might decide - * to just return null, which is a valid implementation. - */ - String GetImage(); - - /** - * Returns an array of characters that make up the suffix of length 'len' for - * the currently matched token. This is used to build up the matched string - * for use in actions in the case of MORE. A simple and inefficient - * implementation of this is as follows : - * - * { - * String t = GetImage(); - * return t.substring(t.length() - len, t.length()).toCharArray(); - * } - */ - char[] GetSuffix(int len); - - /** - * The lexer calls this function to indicate that it is done with the stream - * and hence implementations can free any resources held by this class. - * Again, the body of this function can be just empty and it will not - * affect the lexer's operation. - */ - void Done(); - - - void setTabSize(int i); - int getTabSize(); - boolean getTrackLineColumn(); - void setTrackLineColumn(boolean trackLineColumn); -} -/* (filtered)*/ diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/FastCharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/FastCharStream.java deleted file mode 100644 index 7b206fc1155..00000000000 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/FastCharStream.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.queryparser.flexible.standard.parser; - -import java.io.*; - -/** An efficient implementation of JavaCC's CharStream interface.
Note that - * this does not do line-number counting, but instead keeps track of the - * character position of the token in the input, as required by Lucene's {@link - * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API. - * */ -public final class FastCharStream implements CharStream { - char[] buffer = null; - - int bufferLength = 0; // end of valid chars - int bufferPosition = 0; // next char to read - - int tokenStart = 0; // offset in buffer - int bufferStart = 0; // position in file of buffer - - Reader input; // source of chars - - /** Constructs from a Reader. */ - public FastCharStream(Reader r) { - input = r; - } - - @Override - public final char readChar() throws IOException { - if (bufferPosition >= bufferLength) - refill(); - return buffer[bufferPosition++]; - } - - private final void refill() throws IOException { - int newPosition = bufferLength - tokenStart; - - if (tokenStart == 0) { // token won't fit in buffer - if (buffer == null) { // first time: alloc buffer - buffer = new char[2048]; - } else if (bufferLength == buffer.length) { // grow buffer - char[] newBuffer = new char[buffer.length * 2]; - System.arraycopy(buffer, 0, newBuffer, 0, bufferLength); - buffer = newBuffer; - } - } else { // shift token to front - System.arraycopy(buffer, tokenStart, buffer, 0, newPosition); - } - - bufferLength = newPosition; // update state - bufferPosition = newPosition; - bufferStart += tokenStart; - tokenStart = 0; - - int charsRead = // fill space in buffer - input.read(buffer, newPosition, buffer.length-newPosition); - if (charsRead == -1) - throw new IOException("read past eof"); - else - bufferLength += charsRead; - } - - @Override - public final char BeginToken() throws IOException { - tokenStart = bufferPosition; - return readChar(); - } - - @Override - public final void backup(int amount) { - bufferPosition -= amount; - } - - @Override - public final String GetImage() { - return new String(buffer, tokenStart, bufferPosition - tokenStart); - } - - @Override - public final char[] GetSuffix(int len) { - char[] value = new char[len]; - System.arraycopy(buffer, bufferPosition - len, value, 0, len); - return value; - } - - @Override - public final void Done() { - try { - input.close(); - } catch (IOException e) { - } - } - - /** - * Returns the column position of the character last read. - * @deprecated see #getEndColumn - */ - @Deprecated - @Override - public final int getColumn() { - return bufferStart + bufferPosition; - } - /** - * @deprecated see #getEndLine - */ - @Deprecated - @Override - public final int getLine() { - return 1; - } - @Override - public final int getEndColumn() { - return bufferStart + bufferPosition; - } - @Override - public final int getEndLine() { - return 1; - } - @Override - public final int getBeginColumn() { - return bufferStart + tokenStart; - } - @Override - public final int getBeginLine() { - return 1; - } - - @Override - public void setTabSize(int i) { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public int getTabSize() { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public boolean getTrackLineColumn() { - return false; - } - - @Override - public void setTrackLineColumn(boolean trackLineColumn) { - throw new RuntimeException("Line/Column tracking not implemented."); - } -} diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java index 66500fac40d..4a5667f2001 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java @@ -41,6 +41,8 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QuotedFieldQueryNode; import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; /** * Parser for the standard Lucene syntax @@ -747,17 +749,6 @@ if (boost != null) { finally { jj_save(2, xla); } } - private boolean jj_3R_26() - { - Token xsp; - xsp = jj_scanpos; - if (jj_3_2()) { - jj_scanpos = xsp; - if (jj_3_3()) return true; - } - return false; - } - private boolean jj_3R_22() { if (jj_scan_token(OR)) return true; @@ -1045,6 +1036,17 @@ if (boost != null) { return false; } + private boolean jj_3R_26() + { + Token xsp; + xsp = jj_scanpos; + if (jj_3_2()) { + jj_scanpos = xsp; + if (jj_3_3()) return true; + } + return false; + } + /** Generated Token Manager. */ public StandardSyntaxParserTokenManager token_source; /** Current token. */ diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj index 69d10847a79..a25de283b51 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj @@ -48,6 +48,8 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QuotedFieldQueryNode; import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; /** * Parser for the standard Lucene syntax diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java index bbfa11f0e01..2860fae683c 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java @@ -1,6 +1,7 @@ /* StandardSyntaxParserTokenManager.java */ /* Generated By:JavaCC: Do not edit this line. StandardSyntaxParserTokenManager.java */ package org.apache.lucene.queryparser.flexible.standard.parser; + import org.apache.lucene.queryparser.charstream.CharStream; /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -37,6 +38,8 @@ package org.apache.lucene.queryparser.flexible.standard.parser; + + diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java deleted file mode 100644 index 4481de6c663..00000000000 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java +++ /dev/null @@ -1,120 +0,0 @@ -/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 7.0 */ -/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ -package org.apache.lucene.queryparser.surround.parser; - -/** - * This interface describes a character stream that maintains line and - * column number positions of the characters. It also has the capability - * to backup the stream to some extent. An implementation of this - * interface is used in the TokenManager implementation generated by - * JavaCCParser. - * - * All the methods except backup can be implemented in any fashion. backup - * needs to be implemented correctly for the correct operation of the lexer. - * Rest of the methods are all used to get information like line number, - * column number and the String that constitutes a token and are not used - * by the lexer. Hence their implementation won't affect the generated lexer's - * operation. - */ - -public -interface CharStream { - - /** - * Returns the next character from the selected input. The method - * of selecting the input is the responsibility of the class - * implementing this interface. Can throw any java.io.IOException. - */ - char readChar() throws java.io.IOException; - - @Deprecated - /** - * Returns the column position of the character last read. - * @deprecated - * @see #getEndColumn - */ - int getColumn(); - - @Deprecated - /** - * Returns the line number of the character last read. - * @deprecated - * @see #getEndLine - */ - int getLine(); - - /** - * Returns the column number of the last character for current token (being - * matched after the last call to BeginTOken). - */ - int getEndColumn(); - - /** - * Returns the line number of the last character for current token (being - * matched after the last call to BeginTOken). - */ - int getEndLine(); - - /** - * Returns the column number of the first character for current token (being - * matched after the last call to BeginTOken). - */ - int getBeginColumn(); - - /** - * Returns the line number of the first character for current token (being - * matched after the last call to BeginTOken). - */ - int getBeginLine(); - - /** - * Backs up the input stream by amount steps. Lexer calls this method if it - * had already read some characters, but could not use them to match a - * (longer) token. So, they will be used again as the prefix of the next - * token and it is the implementation's responsibility to do this right. - */ - void backup(int amount); - - /** - * Returns the next character that marks the beginning of the next token. - * All characters must remain in the buffer between two successive calls - * to this method to implement backup correctly. - */ - char BeginToken() throws java.io.IOException; - - /** - * Returns a string made up of characters from the marked token beginning - * to the current buffer position. Implementations have the choice of returning - * anything that they want to. For example, for efficiency, one might decide - * to just return null, which is a valid implementation. - */ - String GetImage(); - - /** - * Returns an array of characters that make up the suffix of length 'len' for - * the currently matched token. This is used to build up the matched string - * for use in actions in the case of MORE. A simple and inefficient - * implementation of this is as follows : - * - * { - * String t = GetImage(); - * return t.substring(t.length() - len, t.length()).toCharArray(); - * } - */ - char[] GetSuffix(int len); - - /** - * The lexer calls this function to indicate that it is done with the stream - * and hence implementations can free any resources held by this class. - * Again, the body of this function can be just empty and it will not - * affect the lexer's operation. - */ - void Done(); - - - void setTabSize(int i); - int getTabSize(); - boolean getTrackLineColumn(); - void setTrackLineColumn(boolean trackLineColumn); -} -/* (filtered)*/ diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java index 34397d215b8..aac680f474a 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java @@ -18,6 +18,8 @@ import org.apache.lucene.queryparser.surround.query.DistanceQuery; import org.apache.lucene.queryparser.surround.query.SrndTermQuery; import org.apache.lucene.queryparser.surround.query.SrndPrefixQuery; import org.apache.lucene.queryparser.surround.query.SrndTruncQuery; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; /** * This class is generated by JavaCC. The only method that clients should need diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj index 787ed16bde4..a91800d527b 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj @@ -46,6 +46,8 @@ import org.apache.lucene.queryparser.surround.query.DistanceQuery; import org.apache.lucene.queryparser.surround.query.SrndTermQuery; import org.apache.lucene.queryparser.surround.query.SrndPrefixQuery; import org.apache.lucene.queryparser.surround.query.SrndTruncQuery; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; /** * This class is generated by JavaCC. The only method that clients should need diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java index 0f0d30c13ac..d12f2074387 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java @@ -1,6 +1,9 @@ /* QueryParserTokenManager.java */ /* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */ package org.apache.lucene.queryparser.surround.parser; + import org.apache.lucene.queryparser.charstream.CharStream; + + diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java index 91b3e25314f..2338c1e9bfd 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.charstream.CharStream; import org.apache.lucene.queryparser.classic.QueryParser.Operator; import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration; import org.apache.lucene.queryparser.util.QueryParserTestBase; diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java index 1f9fd91e956..e0fd4baf3bc 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java @@ -45,6 +45,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.charstream.FastCharStream; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.core.nodes.FuzzyQueryNode; @@ -54,7 +55,6 @@ import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessor import org.apache.lucene.queryparser.flexible.messages.MessageImpl; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler; import org.apache.lucene.queryparser.flexible.standard.nodes.WildcardQueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.FastCharStream; import org.apache.lucene.queryparser.flexible.standard.parser.ParseException; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.apache.lucene.search.BooleanClause.Occur; diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 97c2ad20362..ebcc7359a63 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -126,6 +126,8 @@ Other Changes * SOLR-14846: Backup/Restore classes no longer take Optional method parameters and will accept nulls instead. (Mike Drob) +* LUCENE-9531: Consolidated CharStream and FastCharStream classes: these have been moved from each query parser + package to org.apache.lucene.queryparser.charstream (Dawid Weiss). Bug Fixes --------------------- diff --git a/solr/core/src/java/org/apache/solr/parser/CharStream.java b/solr/core/src/java/org/apache/solr/parser/CharStream.java deleted file mode 100644 index 2e295cf2443..00000000000 --- a/solr/core/src/java/org/apache/solr/parser/CharStream.java +++ /dev/null @@ -1,120 +0,0 @@ -/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 7.0 */ -/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ -package org.apache.solr.parser; - -/** - * This interface describes a character stream that maintains line and - * column number positions of the characters. It also has the capability - * to backup the stream to some extent. An implementation of this - * interface is used in the TokenManager implementation generated by - * JavaCCParser. - * - * All the methods except backup can be implemented in any fashion. backup - * needs to be implemented correctly for the correct operation of the lexer. - * Rest of the methods are all used to get information like line number, - * column number and the String that constitutes a token and are not used - * by the lexer. Hence their implementation won't affect the generated lexer's - * operation. - */ - -public -interface CharStream { - - /** - * Returns the next character from the selected input. The method - * of selecting the input is the responsibility of the class - * implementing this interface. Can throw any java.io.IOException. - */ - char readChar() throws java.io.IOException; - - @Deprecated - /** - * Returns the column position of the character last read. - * @deprecated - * @see #getEndColumn - */ - int getColumn(); - - @Deprecated - /** - * Returns the line number of the character last read. - * @deprecated - * @see #getEndLine - */ - int getLine(); - - /** - * Returns the column number of the last character for current token (being - * matched after the last call to BeginTOken). - */ - int getEndColumn(); - - /** - * Returns the line number of the last character for current token (being - * matched after the last call to BeginTOken). - */ - int getEndLine(); - - /** - * Returns the column number of the first character for current token (being - * matched after the last call to BeginTOken). - */ - int getBeginColumn(); - - /** - * Returns the line number of the first character for current token (being - * matched after the last call to BeginTOken). - */ - int getBeginLine(); - - /** - * Backs up the input stream by amount steps. Lexer calls this method if it - * had already read some characters, but could not use them to match a - * (longer) token. So, they will be used again as the prefix of the next - * token and it is the implementation's responsibility to do this right. - */ - void backup(int amount); - - /** - * Returns the next character that marks the beginning of the next token. - * All characters must remain in the buffer between two successive calls - * to this method to implement backup correctly. - */ - char BeginToken() throws java.io.IOException; - - /** - * Returns a string made up of characters from the marked token beginning - * to the current buffer position. Implementations have the choice of returning - * anything that they want to. For example, for efficiency, one might decide - * to just return null, which is a valid implementation. - */ - String GetImage(); - - /** - * Returns an array of characters that make up the suffix of length 'len' for - * the currently matched token. This is used to build up the matched string - * for use in actions in the case of MORE. A simple and inefficient - * implementation of this is as follows : - * - * { - * String t = GetImage(); - * return t.substring(t.length() - len, t.length()).toCharArray(); - * } - */ - char[] GetSuffix(int len); - - /** - * The lexer calls this function to indicate that it is done with the stream - * and hence implementations can free any resources held by this class. - * Again, the body of this function can be just empty and it will not - * affect the lexer's operation. - */ - void Done(); - - - void setTabSize(int i); - int getTabSize(); - boolean getTrackLineColumn(); - void setTrackLineColumn(boolean trackLineColumn); -} -/* (filtered)*/ diff --git a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java b/solr/core/src/java/org/apache/solr/parser/FastCharStream.java deleted file mode 100644 index 19ecade0284..00000000000 --- a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.parser; - -import java.io.*; - -/** An efficient implementation of JavaCC's CharStream interface.
Note that - * this does not do line-number counting, but instead keeps track of the - * character position of the token in the input, as required by Lucene's {@link - * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API. - * */ -public final class FastCharStream implements CharStream { - char[] buffer = null; - - int bufferLength = 0; // end of valid chars - int bufferPosition = 0; // next char to read - - int tokenStart = 0; // offset in buffer - int bufferStart = 0; // position in file of buffer - - Reader input; // source of chars - - /** Constructs from a Reader. */ - public FastCharStream(Reader r) { - input = r; - } - - @Override - public final char readChar() throws IOException { - if (bufferPosition >= bufferLength) - refill(); - return buffer[bufferPosition++]; - } - - private final void refill() throws IOException { - int newPosition = bufferLength - tokenStart; - - if (tokenStart == 0) { // token won't fit in buffer - if (buffer == null) { // first time: alloc buffer - buffer = new char[2048]; - } else if (bufferLength == buffer.length) { // grow buffer - char[] newBuffer = new char[buffer.length*2]; - System.arraycopy(buffer, 0, newBuffer, 0, bufferLength); - buffer = newBuffer; - } - } else { // shift token to front - System.arraycopy(buffer, tokenStart, buffer, 0, newPosition); - } - - bufferLength = newPosition; // update state - bufferPosition = newPosition; - bufferStart += tokenStart; - tokenStart = 0; - - int charsRead = // fill space in buffer - input.read(buffer, newPosition, buffer.length-newPosition); - if (charsRead == -1) - throw READ_PAST_EOF; - else - bufferLength += charsRead; - } - - @Override - public final char BeginToken() throws IOException { - tokenStart = bufferPosition; - return readChar(); - } - - /** - * This Exception is used as a signal rather than an exceptional state. - */ - private static final IOException READ_PAST_EOF = new IOException("read past eof"); - - @Override - public final void backup(int amount) { - bufferPosition -= amount; - } - - @Override - public final String GetImage() { - return new String(buffer, tokenStart, bufferPosition - tokenStart); - } - - @Override - public final char[] GetSuffix(int len) { - char[] value = new char[len]; - System.arraycopy(buffer, bufferPosition - len, value, 0, len); - return value; - } - - @Override - public final void Done() { - try { - input.close(); - } catch (IOException e) { - } - } - @Override - @Deprecated - /** - * Returns the column position of the character last read. - * @deprecated - * @see #getEndColumn - */ - public final int getLine() { - return 1; - } - @Override - @Deprecated - /** - * Returns the column position of the character last read. - * @deprecated - * @see #getEndColumn - */ - public final int getColumn() { - return bufferStart + bufferPosition; - } - @Override - public final int getEndColumn() { - return bufferStart + bufferPosition; - } - @Override - public final int getEndLine() { - return 1; - } - @Override - public final int getBeginColumn() { - return bufferStart + tokenStart; - } - @Override - public final int getBeginLine() { - return 1; - } - - @Override - public void setTabSize(int i) { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public int getTabSize() { - throw new RuntimeException("Tab size not implemented."); - } - - @Override - public boolean getTrackLineColumn() { - return false; - } - - @Override - public void setTrackLineColumn(boolean trackLineColumn) { - throw new RuntimeException("Line/Column tracking not implemented."); - } -} diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParser.java b/solr/core/src/java/org/apache/solr/parser/QueryParser.java index 4293879da94..97be79a18e9 100644 --- a/solr/core/src/java/org/apache/solr/parser/QueryParser.java +++ b/solr/core/src/java/org/apache/solr/parser/QueryParser.java @@ -15,6 +15,8 @@ import org.apache.lucene.search.Query; import org.apache.solr.search.SyntaxError; import org.apache.solr.search.QParser; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; public class QueryParser extends SolrQueryParserBase implements QueryParserConstants { /** The default operator for parsing queries. @@ -666,27 +668,6 @@ if (splitOnWhitespace == false) { finally { jj_save(2, xla); } } - private boolean jj_3R_6() - { - return false; - } - - private boolean jj_3R_3() - { - if (jj_scan_token(TERM)) return true; - jj_lookingAhead = true; - jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind); - jj_lookingAhead = false; - if (!jj_semLA || jj_3R_6()) return true; - Token xsp; - if (jj_3R_7()) return true; - while (true) { - xsp = jj_scanpos; - if (jj_3R_7()) { jj_scanpos = xsp; break; } - } - return false; - } - private boolean jj_3_3() { Token xsp; @@ -730,6 +711,27 @@ if (splitOnWhitespace == false) { return false; } + private boolean jj_3R_6() + { + return false; + } + + private boolean jj_3R_3() + { + if (jj_scan_token(TERM)) return true; + jj_lookingAhead = true; + jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind); + jj_lookingAhead = false; + if (!jj_semLA || jj_3R_6()) return true; + Token xsp; + if (jj_3R_7()) return true; + while (true) { + xsp = jj_scanpos; + if (jj_3R_7()) { jj_scanpos = xsp; break; } + } + return false; + } + /** Generated Token Manager. */ public QueryParserTokenManager token_source; /** Current token. */ diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParser.jj b/solr/core/src/java/org/apache/solr/parser/QueryParser.jj index 1da42dfd92f..8da3f81d9fe 100644 --- a/solr/core/src/java/org/apache/solr/parser/QueryParser.jj +++ b/solr/core/src/java/org/apache/solr/parser/QueryParser.jj @@ -38,6 +38,8 @@ import org.apache.lucene.search.Query; import org.apache.solr.search.SyntaxError; import org.apache.solr.search.QParser; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; public class QueryParser extends SolrQueryParserBase { /** The default operator for parsing queries. diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java b/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java index c717057fae8..323a3ef8709 100644 --- a/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java +++ b/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java @@ -1,6 +1,9 @@ /* QueryParserTokenManager.java */ /* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */ package org.apache.solr.parser; + import org.apache.lucene.queryparser.charstream.CharStream; + + diff --git a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java index c05c7582ca7..1a2795cfbaa 100644 --- a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java +++ b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java @@ -66,6 +66,8 @@ import org.apache.solr.search.QParser; import org.apache.solr.search.QueryUtils; import org.apache.solr.search.SolrConstantScoreQuery; import org.apache.solr.search.SyntaxError; +import org.apache.lucene.queryparser.charstream.CharStream; +import org.apache.lucene.queryparser.charstream.FastCharStream; import static org.apache.solr.parser.SolrQueryParserBase.SynonymQueryStyle.AS_SAME_TERM; diff --git a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java index 447b8303967..e4420558d77 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java @@ -26,12 +26,12 @@ import java.util.Map; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.queryparser.charstream.CharStream; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.parser.CharStream; import org.apache.solr.parser.ParseException; import org.apache.solr.parser.SolrQueryParserBase; import org.apache.solr.request.SolrQueryRequest; diff --git a/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java b/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java index 68dd38c10f9..75e11749ba2 100644 --- a/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java +++ b/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java @@ -16,6 +16,7 @@ */ package org.apache.solr.parser; +import org.apache.lucene.queryparser.charstream.CharStream; import org.apache.lucene.search.Query; import org.apache.solr.common.SolrException; import org.apache.solr.request.SolrQueryRequest;