Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9528: cleanup of flexible query parser's grammar (#1879)

This commit is contained in:
Dawid Weiss 2020-09-18 09:38:20 +02:00 committed by GitHub
parent 5ec2bac91c
commit 3a92e1b93e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 965 additions and 1457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/RegexpQueryNode.java
View File

@ -26,10 +26,10 @@ import org.apache.lucene.util.BytesRef;
/**
* A {@link RegexpQueryNode} represents {@link RegexpQuery} query Examples: /[a-z]|[0-9]/
*/
public class RegexpQueryNode extends QueryNodeImpl implements TextableQueryNode,
FieldableNode {
public class RegexpQueryNode extends QueryNodeImpl implements TextableQueryNode, FieldableNode {
private CharSequence text;
private CharSequence field;
/**
* @param field
* - field name
@ -46,6 +46,16 @@ FieldableNode {
this.text = text.subSequence(begin, end);
}
/**
* @param field
* - field name
* @param text
* - value that contains a regular expression
*/
public RegexpQueryNode(CharSequence field, CharSequence text) {
this(field, text, 0, text.length());
}
public BytesRef textToBytesRef() {
return new BytesRef(text);
}

1237
lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java
View File

File diff suppressed because it is too large Load Diff

793
lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj
View File

@ -1,9 +1,13 @@
options {
STATIC=false;
JAVA_UNICODE_ESCAPE=true;
USER_CHAR_STREAM=true;
IGNORE_CASE=false;
JDK_VERSION="1.5";
STATIC = false;
JAVA_UNICODE_ESCAPE = true;
USER_CHAR_STREAM = true;
IGNORE_CASE = false;
JDK_VERSION = "1.8";
// FORCE_LA_CHECK = true;
// DEBUG_LOOKAHEAD = true;
// DEBUG_PARSER = true;
}
PARSER_BEGIN(StandardSyntaxParser)
@ -27,13 +31,15 @@ package org.apache.lucene.queryparser.flexible.standard.parser;
*/
import java.io.StringReader;
import java.util.Vector;
import java.util.Arrays;
import java.io.Reader;
import java.util.Collections;
import java.util.ArrayList;
import org.apache.lucene.queryparser.flexible.messages.Message;
import org.apache.lucene.queryparser.flexible.messages.MessageImpl;
import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;
import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages;
import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser;
import org.apache.lucene.queryparser.flexible.core.nodes.AndQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.BoostQueryNode;
@ -42,464 +48,441 @@ import org.apache.lucene.queryparser.flexible.core.nodes.FuzzyQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode;
import org.apache.lucene.queryparser.flexible.standard.nodes.RegexpQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.SlopQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.QuotedFieldQueryNode;
import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser;
import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode;
import org.apache.lucene.queryparser.flexible.standard.nodes.RegexpQueryNode;
import org.apache.lucene.queryparser.charstream.CharStream;
import org.apache.lucene.queryparser.charstream.FastCharStream;
import static org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxImpl.discardEscapeChar;
/**
* Parser for the standard Lucene syntax
*/
public class StandardSyntaxParser implements SyntaxParser {
// syntax parser constructor
public StandardSyntaxParser() {
this(new FastCharStream(new StringReader("")));
public StandardSyntaxParser() {
this(new FastCharStream(Reader.nullReader()));
}
/** Parses a query string, returning a {@link org.apache.lucene.queryparser.flexible.core.nodes.QueryNode}.
* @param query the query string to be parsed.
* @throws ParseException if the parsing fails
*/
public QueryNode parse(CharSequence query, CharSequence field) throws QueryNodeParseException {
ReInit(new FastCharStream(new StringReader(query.toString())));
try {
// TopLevelQuery is a Query followed by the end-of-input (EOF)
QueryNode querynode = TopLevelQuery(field);
return querynode;
}
catch (ParseException tme) {
tme.setQuery(query);
throw tme;
}
catch (Error tme) {
Message message = new MessageImpl(QueryParserMessages.INVALID_SYNTAX_CANNOT_PARSE, query, tme.getMessage());
QueryNodeParseException e = new QueryNodeParseException(tme);
e.setQuery(query);
e.setNonLocalizedMessage(message);
throw e;
}
}
}
/**
* Parses a query string, returning a {@link org.apache.lucene.queryparser.flexible.core.nodes.QueryNode}.
* @param query the query string to be parsed.
* @throws ParseException if the parsing fails
*/
public QueryNode parse(CharSequence query, CharSequence field) throws QueryNodeParseException {
ReInit(new FastCharStream(new StringReader(query.toString())));
try {
return TopLevelQuery(field);
} catch (ParseException tme) {
tme.setQuery(query);
throw tme;
} catch (Error tme) {
Message message = new MessageImpl(QueryParserMessages.INVALID_SYNTAX_CANNOT_PARSE, query, tme.getMessage());
QueryNodeParseException e = new QueryNodeParseException(tme);
e.setQuery(query);
e.setNonLocalizedMessage(message);
throw e;
}
}
}
PARSER_END(StandardSyntaxParser)
/* ***************** */
/* Token Definitions */
/* ***************** */
// Token definitions.
<*> TOKEN : {
<#_NUM_CHAR: ["0"-"9"] >
// every character that follows a backslash is considered as an escaped character
| <#_ESCAPED_CHAR: "\\" ~[] >
| <#_TERM_START_CHAR: ( ~[ " ", "\t", "\n", "\r", "\u3000", "+", "-", "!", "(", ")", ":", "^",
"<", ">", "=", "[", "]", "\"", "{", "}", "~", "\\", "/" ]
| <_ESCAPED_CHAR> ) >
| <#_TERM_CHAR: ( <_TERM_START_CHAR> | <_ESCAPED_CHAR> | "-" | "+" ) >
| <#_WHITESPACE: ( " " | "\t" | "\n" | "\r" | "\u3000") >
| <#_QUOTED_CHAR: ( ~[ "\"", "\\" ] | <_ESCAPED_CHAR> ) >
<#_NUM_CHAR: ["0"-"9"] >
// Every character that follows a backslash is considered as an escaped character
| <#_ESCAPED_CHAR: "\\" ~[] >
| <#_TERM_START_CHAR: ( ~[ " ", "\t", "\n", "\r", "\u3000", "+", "-", "!", "(", ")", ":", "^",
"<", ">", "=", "[", "]", "\"", "{", "}", "~", "\\", "/" ]
| <_ESCAPED_CHAR> ) >
| <#_TERM_CHAR: ( <_TERM_START_CHAR> | <_ESCAPED_CHAR> | "-" | "+" ) >
| <#_WHITESPACE: ( " " | "\t" | "\n" | "\r" | "\u3000") >
| <#_QUOTED_CHAR: ( ~[ "\"", "\\" ] | <_ESCAPED_CHAR> ) >
}
<DEFAULT, Range> SKIP : {
< <_WHITESPACE>>
< <_WHITESPACE> >
}
<DEFAULT> TOKEN : {
<AND: ("AND" | "&&") >
| <OR: ("OR" | "||") >
| <NOT: ("NOT" | "!") >
| <PLUS: "+" >
| <MINUS: "-" >
| <LPAREN: "(" >
| <RPAREN: ")" >
| <OP_COLON: ":" >
| <OP_EQUAL: "=" >
| <OP_LESSTHAN: "<" >
| <OP_LESSTHANEQ: "<=" >
| <OP_MORETHAN: ">" >
| <OP_MORETHANEQ: ">=" >
| <CARAT: "^" > : Boost
| <QUOTED: "\"" (<_QUOTED_CHAR>)* "\"">
| <TERM: <_TERM_START_CHAR> (<_TERM_CHAR>)* >
| <FUZZY_SLOP: "~" ( (<_NUM_CHAR>)+ ( "." (<_NUM_CHAR>)+ )? )? >
| <REGEXPTERM: "/" (~[ "/" ] | "\\/" )* "/" >
| <RANGEIN_START: "[" > : Range
| <RANGEEX_START: "{" > : Range
}
<Boost> TOKEN : {
<NUMBER: (<_NUM_CHAR>)+ ( "." (<_NUM_CHAR>)+ )? > : DEFAULT
<AND: ("AND" | "&&") >
| <OR: ("OR" | "||") >
| <NOT: ("NOT" | "!") >
| <PLUS: "+" >
| <MINUS: "-" >
| <LPAREN: "(" >
| <RPAREN: ")" >
| <OP_COLON: ":" >
| <OP_EQUAL: "=" >
| <OP_LESSTHAN: "<" >
| <OP_LESSTHANEQ: "<=" >
| <OP_MORETHAN: ">" >
| <OP_MORETHANEQ: ">=" >
| <CARAT: "^" >
| <TILDE: "~" >
| <QUOTED: "\"" (<_QUOTED_CHAR>)* "\"">
| <NUMBER: (<_NUM_CHAR>)+ ( "." (<_NUM_CHAR>)+ )? >
| <TERM: <_TERM_START_CHAR> (<_TERM_CHAR>)* >
| <REGEXPTERM: "/" (~[ "/" ] | "\\/" )* "/" >
| <RANGEIN_START: "[" > : Range
| <RANGEEX_START: "{" > : Range
}
<Range> TOKEN : {
<RANGE_TO: "TO">
| <RANGEIN_END: "]"> : DEFAULT
| <RANGEEX_END: "}"> : DEFAULT
| <RANGE_QUOTED: "\"" (~["\""] | "\\\"")+ "\"">
| <RANGE_GOOP: (~[ " ", "]", "}" ])+ >
<RANGE_TO: "TO">
| <RANGEIN_END: "]"> : DEFAULT
| <RANGEEX_END: "}"> : DEFAULT
| <RANGE_QUOTED: "\"" (~["\""] | "\\\"")+ "\"">
| <RANGE_GOOP: (~[ " ", "]", "}" ])+ >
}
ModifierQueryNode.Modifier Modifiers() : {
ModifierQueryNode.Modifier ret = ModifierQueryNode.Modifier.MOD_NONE;
}
{
[
<PLUS> { ret = ModifierQueryNode.Modifier.MOD_REQ; }
| <MINUS> { ret = ModifierQueryNode.Modifier.MOD_NOT; }
| <NOT> { ret = ModifierQueryNode.Modifier.MOD_NOT; }
]
{ return ret; }
}
// This makes sure that there is no garbage after the query string
QueryNode TopLevelQuery(CharSequence field) :
// Non-terminal production rules.
/**
* The top-level rule ensures that there is no garbage after the query string.
*
* <pre>{@code
* TopLevelQuery ::= Query <EOF>
* }</pre>
*/
public QueryNode TopLevelQuery(CharSequence field) :
{
QueryNode q;
}
{
q=Query(field) <EOF>
{
return q;
q = Query(field) <EOF> {
return q;
}
}
// These changes were made to introduce operator precedence:
// - Clause() now returns a QueryNode.
// - The modifiers are consumed by Clause() and returned as part of the QueryNode Object
// - Query does not consume conjunctions (AND, OR) anymore.
// - This is now done by two new non-terminals: ConjClause and DisjClause
// The parse tree looks similar to this:
// Query ::= DisjQuery ( DisjQuery )*
// DisjQuery ::= ConjQuery ( OR ConjQuery )*
// ConjQuery ::= Clause ( AND Clause )*
// Clause ::= [ Modifier ] ...
QueryNode Query(CharSequence field) :
{
Vector<QueryNode> clauses = null;
QueryNode c, first=null;
/**
* A query consists of one or more disjunction queries (solves operator precedence).
* <pre>{@code
* Query ::= DisjQuery ( DisjQuery )*
* DisjQuery ::= ConjQuery ( OR ConjQuery )*
* ConjQuery ::= ModClause ( AND ModClause )*
* }</pre>
*/
private QueryNode Query(CharSequence field) : {
ArrayList<QueryNode> clauses = new ArrayList<QueryNode>();
QueryNode node;
}
{
first=DisjQuery(field)
(
c=DisjQuery(field)
{
if (clauses == null) {
clauses = new Vector<QueryNode>();
clauses.addElement(first);
}
clauses.addElement(c);
}
)*
{
if (clauses != null) {
return new BooleanQueryNode(clauses);
} else {
// Handle the case of a "pure" negation query which
// needs to be wrapped as a boolean query, otherwise
// the returned result drops the negation.
if (first instanceof ModifierQueryNode) {
ModifierQueryNode m = (ModifierQueryNode) first;
if (m.getModifier() == ModifierQueryNode.Modifier.MOD_NOT) {
return new BooleanQueryNode(Arrays.asList(m));
}
}
return first;
( node = DisjQuery(field) { clauses.add(node); } )+
{
// Handle the case of a "pure" negation query which
// needs to be wrapped as a boolean query, otherwise
// the returned result drops the negation.
if (clauses.size() == 1) {
QueryNode first = clauses.get(0);
if (first instanceof ModifierQueryNode
&& ((ModifierQueryNode) first).getModifier() == ModifierQueryNode.Modifier.MOD_NOT) {
clauses.set(0, new BooleanQueryNode(Collections.singletonList(first)));
}
}
}
QueryNode DisjQuery(CharSequence field) : {
QueryNode first, c;
Vector<QueryNode> clauses = null;
}
{
first = ConjQuery(field)
(
<OR> c=ConjQuery(field)
{
if (clauses == null) {
clauses = new Vector<QueryNode>();
clauses.addElement(first);
}
clauses.addElement(c);
}
)*
{
if (clauses != null) {
return new OrQueryNode(clauses);
} else {
return first;
}
return clauses.size() == 1 ? clauses.get(0) : new BooleanQueryNode(clauses);
}
}
QueryNode ConjQuery(CharSequence field) : {
QueryNode first, c;
Vector<QueryNode> clauses = null;
/**
* A disjoint clause consists of one or more conjunction clauses.
* <pre>{@code
* DisjQuery ::= ConjQuery ( OR ConjQuery )*
* }</pre>
*/
private QueryNode DisjQuery(CharSequence field) : {
ArrayList<QueryNode> clauses = new ArrayList<QueryNode>();
QueryNode node;
}
{
first = ModClause(field)
(
<AND> c=ModClause(field)
{
if (clauses == null) {
clauses = new Vector<QueryNode>();
clauses.addElement(first);
}
clauses.addElement(c);
}
)*
node = ConjQuery(field) { clauses.add(node); }
( <OR> node = ConjQuery(field) { clauses.add(node); } )*
{
if (clauses != null) {
return new AndQueryNode(clauses);
} else {
return first;
}
return clauses.size() == 1 ? clauses.get(0) : new OrQueryNode(clauses);
}
}
// QueryNode Query(CharSequence field) :
// {
// List clauses = new ArrayList();
// List modifiers = new ArrayList();
// QueryNode q, firstQuery=null;
// ModifierQueryNode.Modifier mods;
// int conj;
// }
// {
// mods=Modifiers() q=Clause(field)
// {
// if (mods == ModifierQueryNode.Modifier.MOD_NONE) firstQuery=q;
//
// // do not create modifier nodes with MOD_NONE
// if (mods != ModifierQueryNode.Modifier.MOD_NONE) {
// q = new ModifierQueryNode(q, mods);
// }
// clauses.add(q);
// }
// (
// conj=Conjunction() mods=Modifiers() q=Clause(field)
// {
// // do not create modifier nodes with MOD_NONE
// if (mods != ModifierQueryNode.Modifier.MOD_NONE) {
// q = new ModifierQueryNode(q, mods);
// }
// clauses.add(q);
// //TODO: figure out what to do with AND and ORs
// }
// )*
// {
// if (clauses.size() == 1 && firstQuery != null)
// return firstQuery;
// else {
// return new BooleanQueryNode(clauses);
// }
// }
// }
QueryNode ModClause(CharSequence field) : {
QueryNode q;
ModifierQueryNode.Modifier mods;
/**
* A conjunction clause consists of one or more modifier-clause pairs.
* <pre>{@code
* ConjQuery ::= ModClause ( AND ModClause )*
* }</pre>
*/
private QueryNode ConjQuery(CharSequence field) : {
ArrayList<QueryNode> clauses = new ArrayList<QueryNode>();
QueryNode node;
}
{
mods=Modifiers() q= Clause(field) {
if (mods != ModifierQueryNode.Modifier.MOD_NONE) {
q = new ModifierQueryNode(q, mods);
}
return q;
}
node = ModClause(field) { clauses.add(node); }
( <AND> node = ModClause(field) { clauses.add(node); } )*
{
return clauses.size() == 1 ? clauses.get(0) : new AndQueryNode(clauses);
}
}
QueryNode Clause(CharSequence field) : {
/**
* A modifier-atomic clause pair.
* <pre>{@code
* ModClause ::= (Modifier)? Clause
* }</pre>
*/
private QueryNode ModClause(CharSequence field) : {
QueryNode q;
Token fieldToken=null, boost=null, operator=null, term=null;
ModifierQueryNode.Modifier modifier = ModifierQueryNode.Modifier.MOD_NONE;
}
{
( <PLUS> { modifier = ModifierQueryNode.Modifier.MOD_REQ; }
| (<MINUS> | <NOT>) { modifier = ModifierQueryNode.Modifier.MOD_NOT; }
)?
q = Clause(field)
{
if (modifier != ModifierQueryNode.Modifier.MOD_NONE) {
q = new ModifierQueryNode(q, modifier);
}
return q;
}
}
/**
* An atomic clause consists of a field range expression, a potentially
* field-qualified term or a group.
*
* <pre>{@code
* Clause ::= FieldRangeExpr
* | (FieldName (':' | '='))? (Term | GroupingExpr)
* }</pre>
*/
private QueryNode Clause(CharSequence field) : {
QueryNode q;
}
{
(
LOOKAHEAD(2) q = FieldRangeExpr(field)
| (LOOKAHEAD(2) field = FieldName() ( <OP_COLON> | <OP_EQUAL> ))? ( q = Term(field) | q = GroupingExpr(field))
)
{
return q;
}
}
/**
* A field name. This utility method strips escape characters from field names.
*/
private CharSequence FieldName() : {
Token name;
}
{
name = <TERM> { return discardEscapeChar(name.image); }
}
/**
* An grouping expression is a Query with potential boost applied to it.
*
* <pre>{@code
* GroupingExpr ::= '(' Query ')' ('^' <NUMBER>)?
* }</pre>
*/
private GroupQueryNode GroupingExpr(CharSequence field) : {
QueryNode q;
Token boost;
}
{
<LPAREN> q = Query(field) <RPAREN> (q = Boost(q))?
{
return new GroupQueryNode(q);
}
}
/**
* Score boost modifier.
*
* <pre>{@code
* Boost ::= '^' <NUMBER>
* }</pre>
*/
private QueryNode Boost(QueryNode node) : {
Token boost;
}
{
<CARAT> boost = <NUMBER>
{
return node == null ? node : new BoostQueryNode(node, Float.parseFloat(boost.image));
}
}
/**
* Fuzzy term modifier.
*
* <pre>{@code
* Fuzzy ::= '~' <NUMBER>?
* }</pre>
*/
private QueryNode FuzzyOp(CharSequence field, Token term, QueryNode node) : {
Token similarity = null;
}
{
<TILDE> (LOOKAHEAD(2) similarity = <NUMBER>)?
{
float fms = org.apache.lucene.search.FuzzyQuery.defaultMaxEdits;
if (similarity != null) {
fms = Float.parseFloat(similarity.image);
if (fms < 0.0f) {
throw new ParseException(new MessageImpl(QueryParserMessages.INVALID_SYNTAX_FUZZY_LIMITS));
} else if (fms >= 1.0f && fms != (int) fms) {
throw new ParseException(new MessageImpl(QueryParserMessages.INVALID_SYNTAX_FUZZY_EDITS));
}
}
return new FuzzyQueryNode(field, discardEscapeChar(term.image), fms, term.beginColumn, term.endColumn);
}
}
/**
* A field range expression selects all field values larger/ smaller (or equal) than a given one.
* <pre>{@code
* FieldRangeExpr ::= FieldName ('<' | '>' | '<=' | '>=') (<TERM> | <QUOTED> | <NUMBER>)
* }</pre>
*/
private TermRangeQueryNode FieldRangeExpr(CharSequence field) : {
Token operator, term;
FieldQueryNode qLower, qUpper;
boolean lowerInclusive, upperInclusive;
boolean group = false;
}
{
(
LOOKAHEAD(3) fieldToken=<TERM> (
( <OP_COLON> | <OP_EQUAL> ) {field=EscapeQuerySyntaxImpl.discardEscapeChar(fieldToken.image);} q=Term(field)
| ( operator=<OP_LESSTHAN> | operator=<OP_LESSTHANEQ> | operator=<OP_MORETHAN> | operator=<OP_MORETHANEQ> ) {field=EscapeQuerySyntaxImpl.discardEscapeChar(fieldToken.image);}( term=<TERM> | term=<QUOTED> | term=<NUMBER> )
{
if (term.kind == QUOTED) {
term.image = term.image.substring(1, term.image.length()-1);
}
switch (operator.kind) {
case OP_LESSTHAN:
lowerInclusive = true;
upperInclusive = false;
qLower = new FieldQueryNode(field,
"*", term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field,
EscapeQuerySyntaxImpl.discardEscapeChar(term.image), term.beginColumn, term.endColumn);
break;
case OP_LESSTHANEQ:
lowerInclusive = true;
upperInclusive = true;
qLower = new FieldQueryNode(field,
"*", term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field,
EscapeQuerySyntaxImpl.discardEscapeChar(term.image), term.beginColumn, term.endColumn);
break;
case OP_MORETHAN:
lowerInclusive = false;
upperInclusive = true;
qLower = new FieldQueryNode(field,
EscapeQuerySyntaxImpl.discardEscapeChar(term.image), term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field,
"*", term.beginColumn, term.endColumn);
break;
case OP_MORETHANEQ:
lowerInclusive = true;
upperInclusive = true;
qLower = new FieldQueryNode(field,
EscapeQuerySyntaxImpl.discardEscapeChar(term.image), term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field,
"*", term.beginColumn, term.endColumn);
break;
default:
throw new Error("Unhandled case: operator="+operator.toString());
}
q = new TermRangeQueryNode(qLower, qUpper, lowerInclusive, upperInclusive);
}
)
| LOOKAHEAD(3) [
LOOKAHEAD(3)
fieldToken=<TERM>
( <OP_COLON> | <OP_EQUAL> ) {field=EscapeQuerySyntaxImpl.discardEscapeChar(fieldToken.image);}
]
(
(q=Term(field))
| (<LPAREN> q=Query(field) <RPAREN> (<CARAT> boost=<NUMBER>)? {group=true;})
)
)
{
if (boost != null) {
float f = (float)1.0;
try {
f = Float.parseFloat(boost.image);
// avoid boosting null queries, such as those caused by stop words
if (q != null) {
q = new BoostQueryNode(q, f);
}
} catch (Exception ignored) {
/* Should this be handled somehow? (defaults to "no boost", if
* boost number is invalid)
*/
}
}
if (group) { q = new GroupQueryNode(q);}
return q;
}
}
QueryNode Term(CharSequence field) : {
Token term, boost=null, fuzzySlop=null, goop1, goop2;
boolean fuzzy = false;
boolean regexp = false;
boolean startInc=false;
boolean endInc=false;
QueryNode q =null;
FieldQueryNode qLower, qUpper;
float defaultMinSimilarity = org.apache.lucene.search.FuzzyQuery.defaultMaxEdits;
}
{
(
(
term=<TERM> { q = new FieldQueryNode(field, EscapeQuerySyntaxImpl.discardEscapeChar(term.image), term.beginColumn, term.endColumn); }
| term=<REGEXPTERM> { regexp=true; }
| term=<NUMBER>
)
[ fuzzySlop=<FUZZY_SLOP> { fuzzy=true; } ]
[ <CARAT> boost=<NUMBER> [ fuzzySlop=<FUZZY_SLOP> { fuzzy=true; } ] ]
{
if (fuzzy) {
float fms = defaultMinSimilarity;
try {
fms = Float.parseFloat(fuzzySlop.image.substring(1));
} catch (Exception ignored) { }
if(fms < 0.0f){
throw new ParseException(new MessageImpl(QueryParserMessages.INVALID_SYNTAX_FUZZY_LIMITS));
} else if (fms >= 1.0f && fms != (int) fms) {
throw new ParseException(new MessageImpl(QueryParserMessages.INVALID_SYNTAX_FUZZY_EDITS));
}
q = new FuzzyQueryNode(field, EscapeQuerySyntaxImpl.discardEscapeChar(term.image), fms, term.beginColumn, term.endColumn);
} else if (regexp) {
String re = term.image.substring(1, term.image.length()-1);
q = new RegexpQueryNode(field, re, 0, re.length());
}
}
| ( ( <RANGEIN_START> {startInc=true;} | <RANGEEX_START> )
( goop1=<RANGE_GOOP>|goop1=<RANGE_QUOTED>|goop1=<RANGE_TO> )
( <RANGE_TO> )
( goop2=<RANGE_GOOP>|goop2=<RANGE_QUOTED>|goop2=<RANGE_TO> )
( <RANGEIN_END> {endInc=true;} | <RANGEEX_END>))
[ <CARAT> boost=<NUMBER> ]
{
if (goop1.kind == RANGE_QUOTED) {
goop1.image = goop1.image.substring(1, goop1.image.length()-1);
}
if (goop2.kind == RANGE_QUOTED) {
goop2.image = goop2.image.substring(1, goop2.image.length()-1);
}
qLower = new FieldQueryNode(field,
EscapeQuerySyntaxImpl.discardEscapeChar(goop1.image), goop1.beginColumn, goop1.endColumn);
qUpper = new FieldQueryNode(field,
EscapeQuerySyntaxImpl.discardEscapeChar(goop2.image), goop2.beginColumn, goop2.endColumn);
q = new TermRangeQueryNode(qLower, qUpper, startInc ? true : false, endInc ? true : false);
}
| term=<QUOTED> {q = new QuotedFieldQueryNode(field, EscapeQuerySyntaxImpl.discardEscapeChar(term.image.substring(1, term.image.length()-1)), term.beginColumn + 1, term.endColumn - 1);}
[ fuzzySlop=<FUZZY_SLOP> ]
[ <CARAT> boost=<NUMBER> ]
{
int phraseSlop = 0;
if (fuzzySlop != null) {
try {
phraseSlop = (int)Float.parseFloat(fuzzySlop.image.substring(1));
q = new SlopQueryNode(q, phraseSlop);
}
catch (Exception ignored) {
/* Should this be handled somehow? (defaults to "no PhraseSlop", if
* slop number is invalid)
*/
}
}
}
)
field = FieldName()
( <OP_LESSTHAN> | <OP_LESSTHANEQ> | <OP_MORETHAN> | <OP_MORETHANEQ>) { operator = token; }
( <TERM> | <QUOTED> | <NUMBER>) { term = token; }
{
if (boost != null) {
float f = (float)1.0;
try {
f = Float.parseFloat(boost.image);
// avoid boosting null queries, such as those caused by stop words
if (q != null) {
q = new BoostQueryNode(q, f);
}
} catch (Exception ignored) {
/* Should this be handled somehow? (defaults to "no boost", if
* boost number is invalid)
*/
}
if (term.kind == QUOTED) {
term.image = term.image.substring(1, term.image.length() - 1);
}
return q;
switch (operator.kind) {
case OP_LESSTHAN:
lowerInclusive = true;
upperInclusive = false;
qLower = new FieldQueryNode(field, "*", term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field, discardEscapeChar(term.image), term.beginColumn, term.endColumn);
break;
case OP_LESSTHANEQ:
lowerInclusive = true;
upperInclusive = true;
qLower = new FieldQueryNode(field, "*", term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field, discardEscapeChar(term.image), term.beginColumn, term.endColumn);
break;
case OP_MORETHAN:
lowerInclusive = false;
upperInclusive = true;
qLower = new FieldQueryNode(field, discardEscapeChar(term.image), term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field, "*", term.beginColumn, term.endColumn);
break;
case OP_MORETHANEQ:
lowerInclusive = true;
upperInclusive = true;
qLower = new FieldQueryNode(field, discardEscapeChar(term.image), term.beginColumn, term.endColumn);
qUpper = new FieldQueryNode(field, "*", term.beginColumn, term.endColumn);
break;
default:
throw new Error("Unhandled case, operator=" + operator);
}
return new TermRangeQueryNode(qLower, qUpper, lowerInclusive, upperInclusive);
}
}
/**
* A term expression.
*
* <pre>{@code
* Term ::= (<TERM> | <NUMBER>) ('~' <NUM>)? ('^' <NUM>)?
* | <REGEXPTERM> ('^' <NUM>)?
* | TermRangeExpr ('^' <NUM>)?
* | QuotedTerm ('^' <NUM>)?
* }</pre>
*/
private QueryNode Term(CharSequence field) : {
QueryNode q;
Token term, fuzzySlop=null;
}
{
(
term = <REGEXPTERM>
{ q = new RegexpQueryNode(field, term.image.substring(1, term.image.length() - 1)); }
| (term = <TERM> | term = <NUMBER>)
{ q = new FieldQueryNode(field, discardEscapeChar(term.image), term.beginColumn, term.endColumn); }
( q = FuzzyOp(field, term, q) )?
| q = TermRangeExpr(field)
| q = QuotedTerm(field)
)
( q = Boost(q) )?
{
return q;
}
}
/**
* A quoted term (phrase).
*
* <pre>{@code
* QuotedTerm ::= <QUOTED> ('~' <NUM>)?
* }</pre>
*/
private QueryNode QuotedTerm(CharSequence field) : {
QueryNode q;
Token term, slop;
}
{
term = <QUOTED>
{
String image = term.image.substring(1, term.image.length() - 1);
q = new QuotedFieldQueryNode(field, discardEscapeChar(image), term.beginColumn + 1, term.endColumn - 1);
}
( <TILDE> slop = <NUMBER> { q = new SlopQueryNode(q, (int) Float.parseFloat(slop.image)); } )?
{
return q;
}
}
/**
* A value range expression.
*
* <pre>{@code
* TermRangeExpr ::= ('[' | '{') <RANGE_START> 'TO' <RANGE_END> (']' | '}')
* }</pre>
*/
private TermRangeQueryNode TermRangeExpr(CharSequence field) : {
Token left, right;
boolean leftInclusive = false;
boolean rightInclusive = false;
}
{
// RANGE_TO can be consumed as range start/end because this needs to be accepted as a valid range:
// [TO TO TO]
(
(<RANGEIN_START> { leftInclusive = true; } | <RANGEEX_START>)
(<RANGE_GOOP> | <RANGE_QUOTED> | <RANGE_TO>) { left = token; }
<RANGE_TO>
(<RANGE_GOOP> | <RANGE_QUOTED> | <RANGE_TO>) { right = token; }
(<RANGEIN_END> { rightInclusive = true; } | <RANGEEX_END>)
)
{
if (left.kind == RANGE_QUOTED) {
left.image = left.image.substring(1, left.image.length() - 1);
}
if (right.kind == RANGE_QUOTED) {
right.image = right.image.substring(1, right.image.length() - 1);
}
FieldQueryNode qLower = new FieldQueryNode(field,
discardEscapeChar(left.image), left.beginColumn, left.endColumn);
FieldQueryNode qUpper = new FieldQueryNode(field,
discardEscapeChar(right.image), right.beginColumn, right.endColumn);
return new TermRangeQueryNode(qLower, qUpper, leftInclusive, rightInclusive);
}
}

24
lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserConstants.java
View File

@ -51,19 +51,19 @@ public interface StandardSyntaxParserConstants {
/** RegularExpression Id. */
int CARAT = 21;
/** RegularExpression Id. */
int QUOTED = 22;
int TILDE = 22;
/** RegularExpression Id. */
int TERM = 23;
int QUOTED = 23;
/** RegularExpression Id. */
int FUZZY_SLOP = 24;
int NUMBER = 24;
/** RegularExpression Id. */
int REGEXPTERM = 25;
int TERM = 25;
/** RegularExpression Id. */
int RANGEIN_START = 26;
int REGEXPTERM = 26;
/** RegularExpression Id. */
int RANGEEX_START = 27;
int RANGEIN_START = 27;
/** RegularExpression Id. */
int NUMBER = 28;
int RANGEEX_START = 28;
/** RegularExpression Id. */
int RANGE_TO = 29;
/** RegularExpression Id. */
@ -76,11 +76,9 @@ public interface StandardSyntaxParserConstants {
int RANGE_GOOP = 33;
/** Lexical state. */
int Boost = 0;
int Range = 0;
/** Lexical state. */
int Range = 1;
/** Lexical state. */
int DEFAULT = 2;
int DEFAULT = 1;
/** Literal token values. */
String[] tokenImage = {
@ -106,13 +104,13 @@ public interface StandardSyntaxParserConstants {
"\">\"",
"\">=\"",
"\"^\"",
"\"~\"",
"<QUOTED>",
"<NUMBER>",
"<TERM>",
"<FUZZY_SLOP>",
"<REGEXPTERM>",
"\"[\"",
"\"{\"",
"<NUMBER>",
"\"TO\"",
"\"]\"",
"\"}\"",

346
lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java
View File

@ -40,6 +40,8 @@ package org.apache.lucene.queryparser.flexible.standard.parser;
@ -50,15 +52,15 @@ public class StandardSyntaxParserTokenManager implements StandardSyntaxParserCon
// (debugStream omitted).
/** Set debug output. */
// (setDebugStream omitted).
private final int jjStopStringLiteralDfa_2(int pos, long active0){
private final int jjStopStringLiteralDfa_1(int pos, long active0){
switch (pos)
{
default :
return -1;
}
}
private final int jjStartNfa_2(int pos, long active0){
return jjMoveNfa_2(jjStopStringLiteralDfa_2(pos, active0), pos + 1);
private final int jjStartNfa_1(int pos, long active0){
return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1);
}
private int jjStopAtPos(int pos, int kind)
{
@ -66,7 +68,7 @@ private int jjStopAtPos(int pos, int kind)
jjmatchedPos = pos;
return pos + 1;
}
private int jjMoveStringLiteralDfa0_2(){
private int jjMoveStringLiteralDfa0_1(){
switch(curChar)
{
case 40:
@ -81,26 +83,28 @@ private int jjMoveStringLiteralDfa0_2(){
return jjStopAtPos(0, 15);
case 60:
jjmatchedKind = 17;
return jjMoveStringLiteralDfa1_2(0x40000L);
return jjMoveStringLiteralDfa1_1(0x40000L);
case 61:
return jjStopAtPos(0, 16);
case 62:
jjmatchedKind = 19;
return jjMoveStringLiteralDfa1_2(0x100000L);
return jjMoveStringLiteralDfa1_1(0x100000L);
case 91:
return jjStopAtPos(0, 26);
return jjStopAtPos(0, 27);
case 94:
return jjStopAtPos(0, 21);
case 123:
return jjStopAtPos(0, 27);
return jjStopAtPos(0, 28);
case 126:
return jjStopAtPos(0, 22);
default :
return jjMoveNfa_2(0, 0);
return jjMoveNfa_1(0, 0);
}
}
private int jjMoveStringLiteralDfa1_2(long active0){
private int jjMoveStringLiteralDfa1_1(long active0){
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) {
jjStopStringLiteralDfa_2(0, active0);
jjStopStringLiteralDfa_1(0, active0);
return 1;
}
switch(curChar)
@ -114,7 +118,7 @@ private int jjMoveStringLiteralDfa1_2(long active0){
default :
break;
}
return jjStartNfa_2(0, active0);
return jjStartNfa_1(0, active0);
}
static final long[] jjbitVec0 = {
0x1L, 0x0L, 0x0L, 0x0L
@ -128,10 +132,10 @@ static final long[] jjbitVec3 = {
static final long[] jjbitVec4 = {
0xfffefffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
};
private int jjMoveNfa_2(int startState, int curPos)
private int jjMoveNfa_1(int startState, int curPos)
{
int startsAt = 0;
jjnewStateCnt = 33;
jjnewStateCnt = 32;
int i = 1;
jjstateSet[0] = startState;
int kind = 0x7fffffff;
@ -149,9 +153,9 @@ private int jjMoveNfa_2(int startState, int curPos)
case 0:
if ((0x8bff54f8ffffd9ffL & l) != 0L)
{
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
}
else if ((0x100002600L & l) != 0L)
{
@ -167,7 +171,13 @@ private int jjMoveNfa_2(int startState, int curPos)
if (kind > 10)
kind = 10;
}
if (curChar == 38)
if ((0x3ff000000000000L & l) != 0L)
{
if (kind > 24)
kind = 24;
{ jjCheckNAddTwoStates(19, 20); }
}
else if (curChar == 38)
jjstateSet[jjnewStateCnt++] = 4;
break;
case 4:
@ -194,58 +204,58 @@ private int jjMoveNfa_2(int startState, int curPos)
{ jjCheckNAddStates(3, 5); }
break;
case 18:
if (curChar == 34 && kind > 22)
kind = 22;
if (curChar == 34 && kind > 23)
kind = 23;
break;
case 19:
if ((0x8bff54f8ffffd9ffL & l) == 0L)
if ((0x3ff000000000000L & l) == 0L)
break;
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
if (kind > 24)
kind = 24;
{ jjCheckNAddTwoStates(19, 20); }
break;
case 20:
if ((0x8bff7cf8ffffd9ffL & l) == 0L)
if (curChar == 46)
{ jjCheckNAdd(21); }
break;
case 21:
if ((0x3ff000000000000L & l) == 0L)
break;
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
if (kind > 24)
kind = 24;
{ jjCheckNAdd(21); }
break;
case 22:
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
if ((0x8bff54f8ffffd9ffL & l) == 0L)
break;
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
break;
case 23:
if ((0x8bff7cf8ffffd9ffL & l) == 0L)
break;
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
break;
case 25:
if ((0x3ff000000000000L & l) == 0L)
break;
if (kind > 24)
kind = 24;
{ jjAddStates(6, 7); }
break;
case 26:
if (curChar == 46)
{ jjCheckNAdd(27); }
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
break;
case 27:
if ((0x3ff000000000000L & l) == 0L)
break;
if (kind > 24)
kind = 24;
{ jjCheckNAdd(27); }
break;
case 28:
case 30:
case 29:
if (curChar == 47)
{ jjCheckNAddStates(0, 2); }
break;
case 29:
case 28:
if ((0xffff7fffffffffffL & l) != 0L)
{ jjCheckNAddStates(0, 2); }
break;
case 32:
if (curChar == 47 && kind > 25)
kind = 25;
case 31:
if (curChar == 47 && kind > 26)
kind = 26;
break;
default : break;
}
@ -261,18 +271,12 @@ private int jjMoveNfa_2(int startState, int curPos)
case 0:
if ((0x97ffffff87ffffffL & l) != 0L)
{
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
}
else if (curChar == 126)
{
if (kind > 24)
kind = 24;
jjstateSet[jjnewStateCnt++] = 25;
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
}
else if (curChar == 92)
{ jjCheckNAdd(22); }
{ jjCheckNAdd(25); }
if (curChar == 78)
jjstateSet[jjnewStateCnt++] = 11;
else if (curChar == 124)
@ -333,40 +337,33 @@ private int jjMoveNfa_2(int startState, int curPos)
case 17:
{ jjCheckNAddStates(3, 5); }
break;
case 19:
case 20:
case 22:
case 23:
if ((0x97ffffff87ffffffL & l) == 0L)
break;
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
break;
case 21:
if (curChar == 92)
{ jjCheckNAddTwoStates(22, 22); }
break;
case 22:
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
break;
case 23:
if (curChar == 92)
{ jjCheckNAdd(22); }
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
break;
case 24:
if (curChar != 126)
break;
if (kind > 24)
kind = 24;
jjstateSet[jjnewStateCnt++] = 25;
if (curChar == 92)
{ jjCheckNAddTwoStates(25, 25); }
break;
case 29:
case 25:
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
break;
case 26:
if (curChar == 92)
{ jjCheckNAdd(25); }
break;
case 28:
{ jjAddStates(0, 2); }
break;
case 31:
case 30:
if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 30;
jjstateSet[jjnewStateCnt++] = 29;
break;
default : break;
}
@ -391,9 +388,9 @@ private int jjMoveNfa_2(int startState, int curPos)
}
if (jjCanMove_2(hiByte, i1, i2, l1, l2))
{
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
}
break;
case 15:
@ -401,22 +398,22 @@ private int jjMoveNfa_2(int startState, int curPos)
if (jjCanMove_1(hiByte, i1, i2, l1, l2))
{ jjCheckNAddStates(3, 5); }
break;
case 19:
case 20:
case 22:
case 23:
if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
break;
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
break;
case 22:
case 25:
if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
break;
if (kind > 23)
kind = 23;
{ jjCheckNAddTwoStates(20, 21); }
if (kind > 25)
kind = 25;
{ jjCheckNAddTwoStates(23, 24); }
break;
case 29:
case 28:
if (jjCanMove_1(hiByte, i1, i2, l1, l2))
{ jjAddStates(0, 2); }
break;
@ -431,96 +428,13 @@ private int jjMoveNfa_2(int startState, int curPos)
kind = 0x7fffffff;
}
++curPos;
if ((i = jjnewStateCnt) == (startsAt = 33 - (jjnewStateCnt = startsAt)))
if ((i = jjnewStateCnt) == (startsAt = 32 - (jjnewStateCnt = startsAt)))
return curPos;
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return curPos; }
}
}
private int jjMoveStringLiteralDfa0_0()
{
return jjMoveNfa_0(0, 0);
}
private int jjMoveNfa_0(int startState, int curPos)
{
int startsAt = 0;
jjnewStateCnt = 3;
int i = 1;
jjstateSet[0] = startState;
int kind = 0x7fffffff;
for (;;)
{
if (++jjround == 0x7fffffff)
ReInitRounds();
if (curChar < 64)
{
long l = 1L << curChar;
do
{
switch(jjstateSet[--i])
{
case 0:
if ((0x3ff000000000000L & l) == 0L)
break;
if (kind > 28)
kind = 28;
{ jjAddStates(8, 9); }
break;
case 1:
if (curChar == 46)
{ jjCheckNAdd(2); }
break;
case 2:
if ((0x3ff000000000000L & l) == 0L)
break;
if (kind > 28)
kind = 28;
{ jjCheckNAdd(2); }
break;
default : break;
}
} while(i != startsAt);
}
else if (curChar < 128)
{
long l = 1L << (curChar & 077);
do
{
switch(jjstateSet[--i])
{
default : break;
}
} while(i != startsAt);
}
else
{
int hiByte = (curChar >> 8);
int i1 = hiByte >> 6;
long l1 = 1L << (hiByte & 077);
int i2 = (curChar & 0xff) >> 6;
long l2 = 1L << (curChar & 077);
do
{
switch(jjstateSet[--i])
{
default : if (i1 == 0 || l1 == 0 || i2 == 0 || l2 == 0) break; else break;
}
} while(i != startsAt);
}
if (kind != 0x7fffffff)
{
jjmatchedKind = kind;
jjmatchedPos = curPos;
kind = 0x7fffffff;
}
++curPos;
if ((i = jjnewStateCnt) == (startsAt = 3 - (jjnewStateCnt = startsAt)))
return curPos;
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return curPos; }
}
}
private final int jjStopStringLiteralDfa_1(int pos, long active0){
private final int jjStopStringLiteralDfa_0(int pos, long active0){
switch (pos)
{
case 0:
@ -534,48 +448,48 @@ private final int jjStopStringLiteralDfa_1(int pos, long active0){
return -1;
}
}
private final int jjStartNfa_1(int pos, long active0){
return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1);
private final int jjStartNfa_0(int pos, long active0){
return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1);
}
private int jjMoveStringLiteralDfa0_1(){
private int jjMoveStringLiteralDfa0_0(){
switch(curChar)
{
case 84:
return jjMoveStringLiteralDfa1_1(0x20000000L);
return jjMoveStringLiteralDfa1_0(0x20000000L);
case 93:
return jjStopAtPos(0, 30);
case 125:
return jjStopAtPos(0, 31);
default :
return jjMoveNfa_1(0, 0);
return jjMoveNfa_0(0, 0);
}
}
private int jjMoveStringLiteralDfa1_1(long active0){
private int jjMoveStringLiteralDfa1_0(long active0){
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) {
jjStopStringLiteralDfa_1(0, active0);
jjStopStringLiteralDfa_0(0, active0);
return 1;
}
switch(curChar)
{
case 79:
if ((active0 & 0x20000000L) != 0L)
return jjStartNfaWithStates_1(1, 29, 6);
return jjStartNfaWithStates_0(1, 29, 6);
break;
default :
break;
}
return jjStartNfa_1(0, active0);
return jjStartNfa_0(0, active0);
}
private int jjStartNfaWithStates_1(int pos, int kind, int state)
private int jjStartNfaWithStates_0(int pos, int kind, int state)
{
jjmatchedKind = kind;
jjmatchedPos = pos;
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return pos + 1; }
return jjMoveNfa_1(state, pos + 1);
return jjMoveNfa_0(state, pos + 1);
}
private int jjMoveNfa_1(int startState, int curPos)
private int jjMoveNfa_0(int startState, int curPos)
{
int startsAt = 0;
jjnewStateCnt = 7;
@ -614,11 +528,11 @@ private int jjMoveNfa_1(int startState, int curPos)
break;
case 2:
if ((0xfffffffbffffffffL & l) != 0L)
{ jjCheckNAddStates(10, 12); }
{ jjCheckNAddStates(6, 8); }
break;
case 3:
if (curChar == 34)
{ jjCheckNAddStates(10, 12); }
{ jjCheckNAddStates(6, 8); }
break;
case 5:
if (curChar == 34 && kind > 32)
@ -651,7 +565,7 @@ private int jjMoveNfa_1(int startState, int curPos)
{ jjCheckNAdd(6); }
break;
case 2:
{ jjAddStates(10, 12); }
{ jjAddStates(6, 8); }
break;
case 4:
if (curChar == 92)
@ -687,7 +601,7 @@ private int jjMoveNfa_1(int startState, int curPos)
break;
case 2:
if (jjCanMove_1(hiByte, i1, i2, l1, l2))
{ jjAddStates(10, 12); }
{ jjAddStates(6, 8); }
break;
case 6:
if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
@ -717,8 +631,8 @@ private int jjMoveNfa_1(int startState, int curPos)
/** Token literal values. */
public static final String[] jjstrLiteralImages = {
"", null, null, null, null, null, null, null, null, null, null, "\53", "\55",
"\50", "\51", "\72", "\75", "\74", "\74\75", "\76", "\76\75", "\136", null, null,
null, null, "\133", "\173", null, "\124\117", "\135", "\175", null, null, };
"\50", "\51", "\72", "\75", "\74", "\74\75", "\76", "\76\75", "\136", "\176", null,
null, null, null, "\133", "\173", "\124\117", "\135", "\175", null, null, };
protected Token jjFillToken()
{
final Token t;
@ -745,7 +659,7 @@ protected Token jjFillToken()
return t;
}
static final int[] jjnextStates = {
29, 31, 32, 15, 16, 18, 25, 26, 0, 1, 2, 4, 5,
28, 30, 31, 15, 16, 18, 2, 4, 5,
};
private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
{
@ -784,8 +698,8 @@ private static final boolean jjCanMove_2(int hiByte, int i1, int i2, long l1, lo
}
}
int curLexState = 2;
int defaultLexState = 2;
int curLexState = 1;
int defaultLexState = 1;
int jjnewStateCnt;
int jjround;
int jjmatchedPos;
@ -824,11 +738,6 @@ public Token getNextToken()
jjmatchedPos = 0;
curPos = jjMoveStringLiteralDfa0_1();
break;
case 2:
jjmatchedKind = 0x7fffffff;
jjmatchedPos = 0;
curPos = jjMoveStringLiteralDfa0_2();
break;
}
if (jjmatchedKind != 0x7fffffff)
{
@ -954,7 +863,7 @@ private void jjCheckNAddStates(int start, int end)
{
int i;
jjround = 0x80000001;
for (i = 33; i-- > 0;)
for (i = 32; i-- > 0;)
jjrounds[i] = 0x80000000;
}
@ -969,7 +878,7 @@ private void jjCheckNAddStates(int start, int end)
/** Switch to specified lex state. */
public void SwitchTo(int lexState)
{
if (lexState >= 3 || lexState < 0)
if (lexState >= 2 || lexState < 0)
throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
else
curLexState = lexState;
@ -978,15 +887,14 @@ private void jjCheckNAddStates(int start, int end)
/** Lexer state names. */
public static final String[] lexStateNames = {
"Boost",
"Range",
"DEFAULT",
};
/** Lex State array. */
public static final int[] jjnewLexState = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1, -1, -1,
-1, 1, 1, 2, -1, 2, 2, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, 0, 0, -1, 1, 1, -1, -1,
};
static final long[] jjtoToken = {
0x3ffffff01L,
@ -1002,8 +910,8 @@ static final long[] jjtoMore = {
};
protected CharStream input_stream;
private final int[] jjrounds = new int[33];
private final int[] jjstateSet = new int[2 * 33];
private final int[] jjrounds = new int[32];
private final int[] jjstateSet = new int[2 * 32];
private final StringBuilder jjimage = new StringBuilder();
private StringBuilder image = jjimage;
private int jjimageLen;

1
lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
View File

@ -301,7 +301,6 @@ public class TestPrecedenceQueryParser extends LuceneTestCase {
assertQueryEquals("term~", null, "term~2");
assertQueryEquals("term~0.7", null, "term~1");
assertQueryEquals("term~^3", null, "(term~2)^3.0");
assertQueryEquals("term^3~", null, "(term~2)^3.0");
assertQueryEquals("term*germ", null, "term*germ");
assertQueryEquals("term*germ^3", null, "(term*germ)^3.0");

6
lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
View File

@ -567,8 +567,6 @@ public class TestQPHelper extends LuceneTestCase {
assertQueryEquals("term~0.7", null, "term~1");
assertQueryEquals("term~^3", null, "(term~2)^3.0");
assertQueryEquals("term^3~", null, "(term~2)^3.0");
assertQueryEquals("term*germ", null, "term*germ");
assertQueryEquals("term*germ^3", null, "(term*germ)^3.0");
@ -585,6 +583,7 @@ public class TestQPHelper extends LuceneTestCase {
assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
assertQueryNodeException("term~1.1"); // value > 1, throws exception
assertQueryNodeException("term^3~"); // Boost must be applied to FuzzyOp.
assertTrue(getQuery("term*germ", null) instanceof WildcardQuery);
@ -1173,6 +1172,9 @@ public class TestQPHelper extends LuceneTestCase {
re = new RegexpQuery(new Term("field", "http~0.5"));
assertEquals(re, qp.parse("field:/http~0.5/", df));
assertEquals(re, qp.parse("/http~0.5/", df));
// fuzzy op doesn't apply to regexps.
assertQueryNodeException("/http/~2");
re = new RegexpQuery(new Term("field", "boo"));
assertEquals(re, qp.parse("field:/boo/", df));

1
lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
View File

@ -459,7 +459,6 @@ public abstract class QueryParserTestBase extends LuceneTestCase {
assertQueryEquals("term~1", null, "term~1");
assertQueryEquals("term~0.7", null, "term~1");
assertQueryEquals("term~^3", null, "(term~2)^3.0");
assertQueryEquals("term^3~", null, "(term~2)^3.0");
assertQueryEquals("term*germ", null, "term*germ");
assertQueryEquals("term*germ^3", null, "(term*germ)^3.0");