Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5806: Extend expression grammar to allow advanced "variables" 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1609337 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Ryan Ernst 2014-07-09 23:18:58 +00:00
parent 4a7b55d258
commit e37d59b4c8
11 changed files with 1277 additions and 513 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -105,6 +105,10 @@ New Features
BBoxSpatialStrategy using most predicates. Sort documents by relative overlap BBoxSpatialStrategy using most predicates. Sort documents by relative overlap
of query areas or just by indexed shape area. (Ryan McKinley, David Smiley) of query areas or just by indexed shape area. (Ryan McKinley, David Smiley)
* LUCENE-5806: Extend expressions grammar to support array access in variables.
Added helper class VariableContext to parse complex variable into pieces.
(Ryan Ernst)
API Changes API Changes
* LUCENE-5752: Simplified Automaton API to be immutable. (Mike McCandless) * LUCENE-5752: Simplified Automaton API to be immutable. (Mike McCandless)

7
lucene/expressions/src/java/org/apache/lucene/expressions/SimpleBindings.java
View File

@ -63,6 +63,11 @@ public final class SimpleBindings extends Bindings {
map.put(sortField.getField(), sortField); map.put(sortField.getField(), sortField);
} }
/**
* Bind a {@link ValueSource} directly to the given name.
*/
public void add(String name, ValueSource source) { map.put(name, source); }
/** /**
* Adds an Expression to the bindings. * Adds an Expression to the bindings.
* <p> * <p>
@ -79,6 +84,8 @@ public final class SimpleBindings extends Bindings {
throw new IllegalArgumentException("Invalid reference '" + name + "'"); throw new IllegalArgumentException("Invalid reference '" + name + "'");
} else if (o instanceof Expression) { } else if (o instanceof Expression) {
return ((Expression)o).getValueSource(this); return ((Expression)o).getValueSource(this);
} else if (o instanceof ValueSource) {
return ((ValueSource)o);
} }
SortField field = (SortField) o; SortField field = (SortField) o;
switch(field.getType()) { switch(field.getType()) {

39
lucene/expressions/src/java/org/apache/lucene/expressions/js/Javascript.g
View File

@ -309,11 +309,11 @@ unary_operator
postfix postfix
: primary : primary
| NAMESPACE_ID arguments -> ^(AT_CALL NAMESPACE_ID arguments?) | VARIABLE arguments -> ^(AT_CALL VARIABLE arguments?)
; ;
primary primary
: NAMESPACE_ID : VARIABLE
| numeric | numeric
| AT_LPAREN! conditional AT_RPAREN! | AT_LPAREN! conditional AT_RPAREN!
; ;
@ -330,8 +330,19 @@ numeric
// * Lexer Rules // * Lexer Rules
// *********************************************************************** // ***********************************************************************
NAMESPACE_ID VARIABLE
: ID (AT_DOT ID)* : OBJECT (AT_DOT OBJECT)*
;
fragment
OBJECT
: ID ARRAY*
;
fragment
ARRAY
: '[' STRING ']'
| '[' DECIMALINTEGER ']'
; ;
fragment fragment
@ -339,6 +350,26 @@ ID
: ('a'..'z'|'A'..'Z'|'_'|'$') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|'$')* : ('a'..'z'|'A'..'Z'|'_'|'$') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|'$')*
; ;
fragment
STRING
: '\'' SINGLE_STRING_CHAR* '\'' { }
| '"' DOUBLE_STRING_CHAR* '"'
;
fragment
SINGLE_STRING_CHAR
: '\\\''
| '\\\\'
| ~('\\'|'\'')
;
fragment
DOUBLE_STRING_CHAR
: '\\"'
| '\\\\'
| ~('\\'|'"')
;
WS WS
: (' '|'\t'|'\n'|'\r')+ {skip();} : (' '|'\t'|'\n'|'\r')+ {skip();}
; ;

139
lucene/expressions/src/java/org/apache/lucene/expressions/js/Javascript.tokens
View File

@ -1,67 +1,72 @@
AT_ADD=4 ARRAY=4
AT_BIT_AND=5 AT_ADD=5
AT_BIT_NOT=6 AT_BIT_AND=6
AT_BIT_OR=7 AT_BIT_NOT=7
AT_BIT_SHL=8 AT_BIT_OR=8
AT_BIT_SHR=9 AT_BIT_SHL=9
AT_BIT_SHU=10 AT_BIT_SHR=10
AT_BIT_XOR=11 AT_BIT_SHU=11
AT_BOOL_AND=12 AT_BIT_XOR=12
AT_BOOL_NOT=13 AT_BOOL_AND=13
AT_BOOL_OR=14 AT_BOOL_NOT=14
AT_CALL=15 AT_BOOL_OR=15
AT_COLON=16 AT_CALL=16
AT_COMMA=17 AT_COLON=17
AT_COMP_EQ=18 AT_COMMA=18
AT_COMP_GT=19 AT_COMP_EQ=19
AT_COMP_GTE=20 AT_COMP_GT=20
AT_COMP_LT=21 AT_COMP_GTE=21
AT_COMP_LTE=22 AT_COMP_LT=22
AT_COMP_NEQ=23 AT_COMP_LTE=23
AT_COND_QUE=24 AT_COMP_NEQ=24
AT_DIVIDE=25 AT_COND_QUE=25
AT_DOT=26 AT_DIVIDE=26
AT_LPAREN=27 AT_DOT=27
AT_MODULO=28 AT_LPAREN=28
AT_MULTIPLY=29 AT_MODULO=29
AT_NEGATE=30 AT_MULTIPLY=30
AT_RPAREN=31 AT_NEGATE=31
AT_SUBTRACT=32 AT_RPAREN=32
DECIMAL=33 AT_SUBTRACT=33
DECIMALDIGIT=34 DECIMAL=34
DECIMALINTEGER=35 DECIMALDIGIT=35
EXPONENT=36 DECIMALINTEGER=36
HEX=37 DOUBLE_STRING_CHAR=37
HEXDIGIT=38 EXPONENT=38
ID=39 HEX=39
NAMESPACE_ID=40 HEXDIGIT=40
OCTAL=41 ID=41
OCTALDIGIT=42 OBJECT=42
WS=43 OCTAL=43
'!'=13 OCTALDIGIT=44
'!='=23 SINGLE_STRING_CHAR=45
'%'=28 STRING=46
'&&'=12 VARIABLE=47
'&'=5 WS=48
'('=27 '!'=14
')'=31 '!='=24
'*'=29 '%'=29
'+'=4 '&&'=13
','=17 '&'=6
'-'=32 '('=28
'.'=26 ')'=32
'/'=25 '*'=30
':'=16 '+'=5
'<'=21 ','=18
'<<'=8 '-'=33
'<='=22 '.'=27
'=='=18 '/'=26
'>'=19 ':'=17
'>='=20 '<'=22
'>>'=9 '<<'=9
'>>>'=10 '<='=23
'?'=24 '=='=19
'^'=11 '>'=20
'|'=7 '>='=21
'||'=14 '>>'=10
'~'=6 '>>>'=11
'?'=25
'^'=12
'|'=8
'||'=15
'~'=7

46
lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
View File

@ -254,9 +254,13 @@ public class JavascriptCompiler {
gen.cast(Type.DOUBLE_TYPE, expected); gen.cast(Type.DOUBLE_TYPE, expected);
break; break;
case JavascriptParser.NAMESPACE_ID: case JavascriptParser.VARIABLE:
int index; int index;
// normalize quotes
text = normalizeQuotes(text);
if (externalsMap.containsKey(text)) { if (externalsMap.containsKey(text)) {
index = externalsMap.get(text); index = externalsMap.get(text);
} else { } else {
@ -492,6 +496,46 @@ public class JavascriptCompiler {
} }
} }
private static String normalizeQuotes(String text) {
StringBuilder out = new StringBuilder(text.length());
boolean inDoubleQuotes = false;
for (int i = 0; i < text.length(); ++i) {
char c = text.charAt(i);
if (c == '\\') {
c = text.charAt(++i);
if (c == '\\') {
out.append('\\'); // re-escape the backslash
}
// no escape for double quote
} else if (c == '\'') {
if (inDoubleQuotes) {
// escape in output
out.append('\\');
} else {
int j = findSingleQuoteStringEnd(text, i);
out.append(text, i, j); // copy up to end quote (leave end for append below)
i = j;
}
} else if (c == '"') {
c = '\''; // change beginning/ending doubles to singles
inDoubleQuotes = !inDoubleQuotes;
}
out.append(c);
}
return out.toString();
}
private static int findSingleQuoteStringEnd(String text, int start) {
++start; // skip beginning
while (text.charAt(start) != '\'') {
if (text.charAt(start) == '\\') {
++start; // blindly consume escape value
}
++start;
}
return start;
}
/** /**
* The default set of functions available to expressions. * The default set of functions available to expressions.
* <p> * <p>

942
lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptLexer.java
View File

File diff suppressed because it is too large Load Diff

240
lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptParser.java
View File

@ -16,56 +16,62 @@ import org.antlr.runtime.tree.*;
@SuppressWarnings("all") @SuppressWarnings("all")
class JavascriptParser extends Parser { class JavascriptParser extends Parser {
public static final String[] tokenNames = new String[] { public static final String[] tokenNames = new String[] {
"<invalid>", "<EOR>", "<DOWN>", "<UP>", "AT_ADD", "AT_BIT_AND", "AT_BIT_NOT", "<invalid>", "<EOR>", "<DOWN>", "<UP>", "ARRAY", "AT_ADD", "AT_BIT_AND",
"AT_BIT_OR", "AT_BIT_SHL", "AT_BIT_SHR", "AT_BIT_SHU", "AT_BIT_XOR", "AT_BOOL_AND", "AT_BIT_NOT", "AT_BIT_OR", "AT_BIT_SHL", "AT_BIT_SHR", "AT_BIT_SHU", "AT_BIT_XOR",
"AT_BOOL_NOT", "AT_BOOL_OR", "AT_CALL", "AT_COLON", "AT_COMMA", "AT_COMP_EQ", "AT_BOOL_AND", "AT_BOOL_NOT", "AT_BOOL_OR", "AT_CALL", "AT_COLON", "AT_COMMA",
"AT_COMP_GT", "AT_COMP_GTE", "AT_COMP_LT", "AT_COMP_LTE", "AT_COMP_NEQ", "AT_COMP_EQ", "AT_COMP_GT", "AT_COMP_GTE", "AT_COMP_LT", "AT_COMP_LTE",
"AT_COND_QUE", "AT_DIVIDE", "AT_DOT", "AT_LPAREN", "AT_MODULO", "AT_MULTIPLY", "AT_COMP_NEQ", "AT_COND_QUE", "AT_DIVIDE", "AT_DOT", "AT_LPAREN", "AT_MODULO",
"AT_NEGATE", "AT_RPAREN", "AT_SUBTRACT", "DECIMAL", "DECIMALDIGIT", "DECIMALINTEGER", "AT_MULTIPLY", "AT_NEGATE", "AT_RPAREN", "AT_SUBTRACT", "DECIMAL", "DECIMALDIGIT",
"EXPONENT", "HEX", "HEXDIGIT", "ID", "NAMESPACE_ID", "OCTAL", "OCTALDIGIT", "DECIMALINTEGER", "DOUBLE_STRING_CHAR", "EXPONENT", "HEX", "HEXDIGIT",
"WS" "ID", "OBJECT", "OCTAL", "OCTALDIGIT", "SINGLE_STRING_CHAR", "STRING",
"VARIABLE", "WS"
}; };
public static final int EOF=-1; public static final int EOF=-1;
public static final int AT_ADD=4; public static final int ARRAY=4;
public static final int AT_BIT_AND=5; public static final int AT_ADD=5;
public static final int AT_BIT_NOT=6; public static final int AT_BIT_AND=6;
public static final int AT_BIT_OR=7; public static final int AT_BIT_NOT=7;
public static final int AT_BIT_SHL=8; public static final int AT_BIT_OR=8;
public static final int AT_BIT_SHR=9; public static final int AT_BIT_SHL=9;
public static final int AT_BIT_SHU=10; public static final int AT_BIT_SHR=10;
public static final int AT_BIT_XOR=11; public static final int AT_BIT_SHU=11;
public static final int AT_BOOL_AND=12; public static final int AT_BIT_XOR=12;
public static final int AT_BOOL_NOT=13; public static final int AT_BOOL_AND=13;
public static final int AT_BOOL_OR=14; public static final int AT_BOOL_NOT=14;
public static final int AT_CALL=15; public static final int AT_BOOL_OR=15;
public static final int AT_COLON=16; public static final int AT_CALL=16;
public static final int AT_COMMA=17; public static final int AT_COLON=17;
public static final int AT_COMP_EQ=18; public static final int AT_COMMA=18;
public static final int AT_COMP_GT=19; public static final int AT_COMP_EQ=19;
public static final int AT_COMP_GTE=20; public static final int AT_COMP_GT=20;
public static final int AT_COMP_LT=21; public static final int AT_COMP_GTE=21;
public static final int AT_COMP_LTE=22; public static final int AT_COMP_LT=22;
public static final int AT_COMP_NEQ=23; public static final int AT_COMP_LTE=23;
public static final int AT_COND_QUE=24; public static final int AT_COMP_NEQ=24;
public static final int AT_DIVIDE=25; public static final int AT_COND_QUE=25;
public static final int AT_DOT=26; public static final int AT_DIVIDE=26;
public static final int AT_LPAREN=27; public static final int AT_DOT=27;
public static final int AT_MODULO=28; public static final int AT_LPAREN=28;
public static final int AT_MULTIPLY=29; public static final int AT_MODULO=29;
public static final int AT_NEGATE=30; public static final int AT_MULTIPLY=30;
public static final int AT_RPAREN=31; public static final int AT_NEGATE=31;
public static final int AT_SUBTRACT=32; public static final int AT_RPAREN=32;
public static final int DECIMAL=33; public static final int AT_SUBTRACT=33;
public static final int DECIMALDIGIT=34; public static final int DECIMAL=34;
public static final int DECIMALINTEGER=35; public static final int DECIMALDIGIT=35;
public static final int EXPONENT=36; public static final int DECIMALINTEGER=36;
public static final int HEX=37; public static final int DOUBLE_STRING_CHAR=37;
public static final int HEXDIGIT=38; public static final int EXPONENT=38;
public static final int ID=39; public static final int HEX=39;
public static final int NAMESPACE_ID=40; public static final int HEXDIGIT=40;
public static final int OCTAL=41; public static final int ID=41;
public static final int OCTALDIGIT=42; public static final int OBJECT=42;
public static final int WS=43; public static final int OCTAL=43;
public static final int OCTALDIGIT=44;
public static final int SINGLE_STRING_CHAR=45;
public static final int STRING=46;
public static final int VARIABLE=47;
public static final int WS=48;
// delegates // delegates
public Parser[] getDelegates() { public Parser[] getDelegates() {
@ -1271,8 +1277,8 @@ class JavascriptParser extends Parser {
case AT_LPAREN: case AT_LPAREN:
case DECIMAL: case DECIMAL:
case HEX: case HEX:
case NAMESPACE_ID:
case OCTAL: case OCTAL:
case VARIABLE:
{ {
alt12=1; alt12=1;
} }
@ -1493,26 +1499,26 @@ class JavascriptParser extends Parser {
// $ANTLR start "postfix" // $ANTLR start "postfix"
// src/java/org/apache/lucene/expressions/js/Javascript.g:310:1: postfix : ( primary | NAMESPACE_ID arguments -> ^( AT_CALL NAMESPACE_ID ( arguments )? ) ); // src/java/org/apache/lucene/expressions/js/Javascript.g:310:1: postfix : ( primary | VARIABLE arguments -> ^( AT_CALL VARIABLE ( arguments )? ) );
public final JavascriptParser.postfix_return postfix() throws RecognitionException { public final JavascriptParser.postfix_return postfix() throws RecognitionException {
JavascriptParser.postfix_return retval = new JavascriptParser.postfix_return(); JavascriptParser.postfix_return retval = new JavascriptParser.postfix_return();
retval.start = input.LT(1); retval.start = input.LT(1);
CommonTree root_0 = null; CommonTree root_0 = null;
Token NAMESPACE_ID47=null; Token VARIABLE47=null;
ParserRuleReturnScope primary46 =null; ParserRuleReturnScope primary46 =null;
ParserRuleReturnScope arguments48 =null; ParserRuleReturnScope arguments48 =null;
CommonTree NAMESPACE_ID47_tree=null; CommonTree VARIABLE47_tree=null;
RewriteRuleTokenStream stream_NAMESPACE_ID=new RewriteRuleTokenStream(adaptor,"token NAMESPACE_ID"); RewriteRuleTokenStream stream_VARIABLE=new RewriteRuleTokenStream(adaptor,"token VARIABLE");
RewriteRuleSubtreeStream stream_arguments=new RewriteRuleSubtreeStream(adaptor,"rule arguments"); RewriteRuleSubtreeStream stream_arguments=new RewriteRuleSubtreeStream(adaptor,"rule arguments");
try { try {
// src/java/org/apache/lucene/expressions/js/Javascript.g:311:5: ( primary | NAMESPACE_ID arguments -> ^( AT_CALL NAMESPACE_ID ( arguments )? ) ) // src/java/org/apache/lucene/expressions/js/Javascript.g:311:5: ( primary | VARIABLE arguments -> ^( AT_CALL VARIABLE ( arguments )? ) )
int alt14=2; int alt14=2;
int LA14_0 = input.LA(1); int LA14_0 = input.LA(1);
if ( (LA14_0==NAMESPACE_ID) ) { if ( (LA14_0==VARIABLE) ) {
int LA14_1 = input.LA(2); int LA14_1 = input.LA(2);
if ( (LA14_1==EOF||(LA14_1 >= AT_ADD && LA14_1 <= AT_BIT_AND)||(LA14_1 >= AT_BIT_OR && LA14_1 <= AT_BOOL_AND)||LA14_1==AT_BOOL_OR||(LA14_1 >= AT_COLON && LA14_1 <= AT_DIVIDE)||(LA14_1 >= AT_MODULO && LA14_1 <= AT_MULTIPLY)||(LA14_1 >= AT_RPAREN && LA14_1 <= AT_SUBTRACT)) ) { if ( (LA14_1==EOF||(LA14_1 >= AT_ADD && LA14_1 <= AT_BIT_AND)||(LA14_1 >= AT_BIT_OR && LA14_1 <= AT_BOOL_AND)||LA14_1==AT_BOOL_OR||(LA14_1 >= AT_COLON && LA14_1 <= AT_DIVIDE)||(LA14_1 >= AT_MODULO && LA14_1 <= AT_MULTIPLY)||(LA14_1 >= AT_RPAREN && LA14_1 <= AT_SUBTRACT)) ) {
alt14=1; alt14=1;
@ -1560,10 +1566,10 @@ class JavascriptParser extends Parser {
} }
break; break;
case 2 : case 2 :
// src/java/org/apache/lucene/expressions/js/Javascript.g:312:7: NAMESPACE_ID arguments // src/java/org/apache/lucene/expressions/js/Javascript.g:312:7: VARIABLE arguments
{ {
NAMESPACE_ID47=(Token)match(input,NAMESPACE_ID,FOLLOW_NAMESPACE_ID_in_postfix1168); VARIABLE47=(Token)match(input,VARIABLE,FOLLOW_VARIABLE_in_postfix1168);
stream_NAMESPACE_ID.add(NAMESPACE_ID47); stream_VARIABLE.add(VARIABLE47);
pushFollow(FOLLOW_arguments_in_postfix1170); pushFollow(FOLLOW_arguments_in_postfix1170);
arguments48=arguments(); arguments48=arguments();
@ -1571,7 +1577,7 @@ class JavascriptParser extends Parser {
stream_arguments.add(arguments48.getTree()); stream_arguments.add(arguments48.getTree());
// AST REWRITE // AST REWRITE
// elements: arguments, NAMESPACE_ID // elements: VARIABLE, arguments
// token labels: // token labels:
// rule labels: retval // rule labels: retval
// token list labels: // token list labels:
@ -1581,14 +1587,14 @@ class JavascriptParser extends Parser {
RewriteRuleSubtreeStream stream_retval=new RewriteRuleSubtreeStream(adaptor,"rule retval",retval!=null?retval.getTree():null); RewriteRuleSubtreeStream stream_retval=new RewriteRuleSubtreeStream(adaptor,"rule retval",retval!=null?retval.getTree():null);
root_0 = (CommonTree)adaptor.nil(); root_0 = (CommonTree)adaptor.nil();
// 312:30: -> ^( AT_CALL NAMESPACE_ID ( arguments )? ) // 312:26: -> ^( AT_CALL VARIABLE ( arguments )? )
{ {
// src/java/org/apache/lucene/expressions/js/Javascript.g:312:33: ^( AT_CALL NAMESPACE_ID ( arguments )? ) // src/java/org/apache/lucene/expressions/js/Javascript.g:312:29: ^( AT_CALL VARIABLE ( arguments )? )
{ {
CommonTree root_1 = (CommonTree)adaptor.nil(); CommonTree root_1 = (CommonTree)adaptor.nil();
root_1 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(AT_CALL, "AT_CALL"), root_1); root_1 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(AT_CALL, "AT_CALL"), root_1);
adaptor.addChild(root_1, stream_NAMESPACE_ID.nextNode()); adaptor.addChild(root_1, stream_VARIABLE.nextNode());
// src/java/org/apache/lucene/expressions/js/Javascript.g:312:56: ( arguments )? // src/java/org/apache/lucene/expressions/js/Javascript.g:312:48: ( arguments )?
if ( stream_arguments.hasNext() ) { if ( stream_arguments.hasNext() ) {
adaptor.addChild(root_1, stream_arguments.nextTree()); adaptor.addChild(root_1, stream_arguments.nextTree());
} }
@ -1633,28 +1639,28 @@ class JavascriptParser extends Parser {
// $ANTLR start "primary" // $ANTLR start "primary"
// src/java/org/apache/lucene/expressions/js/Javascript.g:315:1: primary : ( NAMESPACE_ID | numeric | AT_LPAREN ! conditional AT_RPAREN !); // src/java/org/apache/lucene/expressions/js/Javascript.g:315:1: primary : ( VARIABLE | numeric | AT_LPAREN ! conditional AT_RPAREN !);
public final JavascriptParser.primary_return primary() throws RecognitionException { public final JavascriptParser.primary_return primary() throws RecognitionException {
JavascriptParser.primary_return retval = new JavascriptParser.primary_return(); JavascriptParser.primary_return retval = new JavascriptParser.primary_return();
retval.start = input.LT(1); retval.start = input.LT(1);
CommonTree root_0 = null; CommonTree root_0 = null;
Token NAMESPACE_ID49=null; Token VARIABLE49=null;
Token AT_LPAREN51=null; Token AT_LPAREN51=null;
Token AT_RPAREN53=null; Token AT_RPAREN53=null;
ParserRuleReturnScope numeric50 =null; ParserRuleReturnScope numeric50 =null;
ParserRuleReturnScope conditional52 =null; ParserRuleReturnScope conditional52 =null;
CommonTree NAMESPACE_ID49_tree=null; CommonTree VARIABLE49_tree=null;
CommonTree AT_LPAREN51_tree=null; CommonTree AT_LPAREN51_tree=null;
CommonTree AT_RPAREN53_tree=null; CommonTree AT_RPAREN53_tree=null;
try { try {
// src/java/org/apache/lucene/expressions/js/Javascript.g:316:5: ( NAMESPACE_ID | numeric | AT_LPAREN ! conditional AT_RPAREN !) // src/java/org/apache/lucene/expressions/js/Javascript.g:316:5: ( VARIABLE | numeric | AT_LPAREN ! conditional AT_RPAREN !)
int alt15=3; int alt15=3;
switch ( input.LA(1) ) { switch ( input.LA(1) ) {
case NAMESPACE_ID: case VARIABLE:
{ {
alt15=1; alt15=1;
} }
@ -1678,14 +1684,14 @@ class JavascriptParser extends Parser {
} }
switch (alt15) { switch (alt15) {
case 1 : case 1 :
// src/java/org/apache/lucene/expressions/js/Javascript.g:316:7: NAMESPACE_ID // src/java/org/apache/lucene/expressions/js/Javascript.g:316:7: VARIABLE
{ {
root_0 = (CommonTree)adaptor.nil(); root_0 = (CommonTree)adaptor.nil();
NAMESPACE_ID49=(Token)match(input,NAMESPACE_ID,FOLLOW_NAMESPACE_ID_in_primary1198); VARIABLE49=(Token)match(input,VARIABLE,FOLLOW_VARIABLE_in_primary1198);
NAMESPACE_ID49_tree = (CommonTree)adaptor.create(NAMESPACE_ID49); VARIABLE49_tree = (CommonTree)adaptor.create(VARIABLE49);
adaptor.addChild(root_0, NAMESPACE_ID49_tree); adaptor.addChild(root_0, VARIABLE49_tree);
} }
break; break;
@ -1776,7 +1782,7 @@ class JavascriptParser extends Parser {
// src/java/org/apache/lucene/expressions/js/Javascript.g:322:18: ( conditional ( AT_COMMA ! conditional )* )? // src/java/org/apache/lucene/expressions/js/Javascript.g:322:18: ( conditional ( AT_COMMA ! conditional )* )?
int alt17=2; int alt17=2;
int LA17_0 = input.LA(1); int LA17_0 = input.LA(1);
if ( (LA17_0==AT_ADD||LA17_0==AT_BIT_NOT||LA17_0==AT_BOOL_NOT||LA17_0==AT_LPAREN||(LA17_0 >= AT_SUBTRACT && LA17_0 <= DECIMAL)||LA17_0==HEX||(LA17_0 >= NAMESPACE_ID && LA17_0 <= OCTAL)) ) { if ( (LA17_0==AT_ADD||LA17_0==AT_BIT_NOT||LA17_0==AT_BOOL_NOT||LA17_0==AT_LPAREN||(LA17_0 >= AT_SUBTRACT && LA17_0 <= DECIMAL)||LA17_0==HEX||LA17_0==OCTAL||LA17_0==VARIABLE) ) {
alt17=1; alt17=1;
} }
switch (alt17) { switch (alt17) {
@ -1906,60 +1912,60 @@ class JavascriptParser extends Parser {
public static final BitSet FOLLOW_conditional_in_expression737 = new BitSet(new long[]{0x0000000000000000L}); public static final BitSet FOLLOW_conditional_in_expression737 = new BitSet(new long[]{0x0000000000000000L});
public static final BitSet FOLLOW_EOF_in_expression739 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_EOF_in_expression739 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_logical_or_in_conditional757 = new BitSet(new long[]{0x0000000001000002L}); public static final BitSet FOLLOW_logical_or_in_conditional757 = new BitSet(new long[]{0x0000000002000002L});
public static final BitSet FOLLOW_AT_COND_QUE_in_conditional760 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_COND_QUE_in_conditional760 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_conditional_in_conditional763 = new BitSet(new long[]{0x0000000000010000L}); public static final BitSet FOLLOW_conditional_in_conditional763 = new BitSet(new long[]{0x0000000000020000L});
public static final BitSet FOLLOW_AT_COLON_in_conditional765 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_COLON_in_conditional765 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_conditional_in_conditional768 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_conditional_in_conditional768 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_logical_and_in_logical_or787 = new BitSet(new long[]{0x0000000000004002L}); public static final BitSet FOLLOW_logical_and_in_logical_or787 = new BitSet(new long[]{0x0000000000008002L});
public static final BitSet FOLLOW_AT_BOOL_OR_in_logical_or790 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_BOOL_OR_in_logical_or790 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_logical_and_in_logical_or793 = new BitSet(new long[]{0x0000000000004002L}); public static final BitSet FOLLOW_logical_and_in_logical_or793 = new BitSet(new long[]{0x0000000000008002L});
public static final BitSet FOLLOW_bitwise_or_in_logical_and812 = new BitSet(new long[]{0x0000000000001002L}); public static final BitSet FOLLOW_bitwise_or_in_logical_and812 = new BitSet(new long[]{0x0000000000002002L});
public static final BitSet FOLLOW_AT_BOOL_AND_in_logical_and815 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_BOOL_AND_in_logical_and815 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_bitwise_or_in_logical_and818 = new BitSet(new long[]{0x0000000000001002L}); public static final BitSet FOLLOW_bitwise_or_in_logical_and818 = new BitSet(new long[]{0x0000000000002002L});
public static final BitSet FOLLOW_bitwise_xor_in_bitwise_or837 = new BitSet(new long[]{0x0000000000000082L}); public static final BitSet FOLLOW_bitwise_xor_in_bitwise_or837 = new BitSet(new long[]{0x0000000000000102L});
public static final BitSet FOLLOW_AT_BIT_OR_in_bitwise_or840 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_BIT_OR_in_bitwise_or840 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_bitwise_xor_in_bitwise_or843 = new BitSet(new long[]{0x0000000000000082L}); public static final BitSet FOLLOW_bitwise_xor_in_bitwise_or843 = new BitSet(new long[]{0x0000000000000102L});
public static final BitSet FOLLOW_bitwise_and_in_bitwise_xor862 = new BitSet(new long[]{0x0000000000000802L}); public static final BitSet FOLLOW_bitwise_and_in_bitwise_xor862 = new BitSet(new long[]{0x0000000000001002L});
public static final BitSet FOLLOW_AT_BIT_XOR_in_bitwise_xor865 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_BIT_XOR_in_bitwise_xor865 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_bitwise_and_in_bitwise_xor868 = new BitSet(new long[]{0x0000000000000802L}); public static final BitSet FOLLOW_bitwise_and_in_bitwise_xor868 = new BitSet(new long[]{0x0000000000001002L});
public static final BitSet FOLLOW_equality_in_bitwise_and888 = new BitSet(new long[]{0x0000000000000022L}); public static final BitSet FOLLOW_equality_in_bitwise_and888 = new BitSet(new long[]{0x0000000000000042L});
public static final BitSet FOLLOW_AT_BIT_AND_in_bitwise_and891 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_BIT_AND_in_bitwise_and891 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_equality_in_bitwise_and894 = new BitSet(new long[]{0x0000000000000022L}); public static final BitSet FOLLOW_equality_in_bitwise_and894 = new BitSet(new long[]{0x0000000000000042L});
public static final BitSet FOLLOW_relational_in_equality913 = new BitSet(new long[]{0x0000000000840002L}); public static final BitSet FOLLOW_relational_in_equality913 = new BitSet(new long[]{0x0000000001080002L});
public static final BitSet FOLLOW_set_in_equality916 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_set_in_equality916 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_relational_in_equality925 = new BitSet(new long[]{0x0000000000840002L}); public static final BitSet FOLLOW_relational_in_equality925 = new BitSet(new long[]{0x0000000001080002L});
public static final BitSet FOLLOW_shift_in_relational944 = new BitSet(new long[]{0x0000000000780002L}); public static final BitSet FOLLOW_shift_in_relational944 = new BitSet(new long[]{0x0000000000F00002L});
public static final BitSet FOLLOW_set_in_relational947 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_set_in_relational947 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_shift_in_relational964 = new BitSet(new long[]{0x0000000000780002L}); public static final BitSet FOLLOW_shift_in_relational964 = new BitSet(new long[]{0x0000000000F00002L});
public static final BitSet FOLLOW_additive_in_shift983 = new BitSet(new long[]{0x0000000000000702L}); public static final BitSet FOLLOW_additive_in_shift983 = new BitSet(new long[]{0x0000000000000E02L});
public static final BitSet FOLLOW_set_in_shift986 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_set_in_shift986 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_additive_in_shift999 = new BitSet(new long[]{0x0000000000000702L}); public static final BitSet FOLLOW_additive_in_shift999 = new BitSet(new long[]{0x0000000000000E02L});
public static final BitSet FOLLOW_multiplicative_in_additive1018 = new BitSet(new long[]{0x0000000100000012L}); public static final BitSet FOLLOW_multiplicative_in_additive1018 = new BitSet(new long[]{0x0000000200000022L});
public static final BitSet FOLLOW_set_in_additive1021 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_set_in_additive1021 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_multiplicative_in_additive1030 = new BitSet(new long[]{0x0000000100000012L}); public static final BitSet FOLLOW_multiplicative_in_additive1030 = new BitSet(new long[]{0x0000000200000022L});
public static final BitSet FOLLOW_unary_in_multiplicative1049 = new BitSet(new long[]{0x0000000032000002L}); public static final BitSet FOLLOW_unary_in_multiplicative1049 = new BitSet(new long[]{0x0000000064000002L});
public static final BitSet FOLLOW_set_in_multiplicative1052 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_set_in_multiplicative1052 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_unary_in_multiplicative1065 = new BitSet(new long[]{0x0000000032000002L}); public static final BitSet FOLLOW_unary_in_multiplicative1065 = new BitSet(new long[]{0x0000000064000002L});
public static final BitSet FOLLOW_postfix_in_unary1084 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_postfix_in_unary1084 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_AT_ADD_in_unary1092 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_ADD_in_unary1092 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_unary_in_unary1095 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_unary_in_unary1095 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_unary_operator_in_unary1103 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_unary_operator_in_unary1103 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_unary_in_unary1106 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_unary_in_unary1106 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_AT_SUBTRACT_in_unary_operator1123 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_AT_SUBTRACT_in_unary_operator1123 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_AT_BIT_NOT_in_unary_operator1135 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_AT_BIT_NOT_in_unary_operator1135 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_AT_BOOL_NOT_in_unary_operator1143 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_AT_BOOL_NOT_in_unary_operator1143 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_primary_in_postfix1160 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_primary_in_postfix1160 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_NAMESPACE_ID_in_postfix1168 = new BitSet(new long[]{0x0000000008000000L}); public static final BitSet FOLLOW_VARIABLE_in_postfix1168 = new BitSet(new long[]{0x0000000010000000L});
public static final BitSet FOLLOW_arguments_in_postfix1170 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_arguments_in_postfix1170 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_NAMESPACE_ID_in_primary1198 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_VARIABLE_in_primary1198 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_numeric_in_primary1206 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_numeric_in_primary1206 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_AT_LPAREN_in_primary1214 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_LPAREN_in_primary1214 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_conditional_in_primary1217 = new BitSet(new long[]{0x0000000080000000L}); public static final BitSet FOLLOW_conditional_in_primary1217 = new BitSet(new long[]{0x0000000100000000L});
public static final BitSet FOLLOW_AT_RPAREN_in_primary1219 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_AT_RPAREN_in_primary1219 = new BitSet(new long[]{0x0000000000000002L});
public static final BitSet FOLLOW_AT_LPAREN_in_arguments1237 = new BitSet(new long[]{0x0000032388002050L}); public static final BitSet FOLLOW_AT_LPAREN_in_arguments1237 = new BitSet(new long[]{0x00008887100040A0L});
public static final BitSet FOLLOW_conditional_in_arguments1241 = new BitSet(new long[]{0x0000000080020000L}); public static final BitSet FOLLOW_conditional_in_arguments1241 = new BitSet(new long[]{0x0000000100040000L});
public static final BitSet FOLLOW_AT_COMMA_in_arguments1244 = new BitSet(new long[]{0x0000032308002050L}); public static final BitSet FOLLOW_AT_COMMA_in_arguments1244 = new BitSet(new long[]{0x00008886100040A0L});
public static final BitSet FOLLOW_conditional_in_arguments1247 = new BitSet(new long[]{0x0000000080020000L}); public static final BitSet FOLLOW_conditional_in_arguments1247 = new BitSet(new long[]{0x0000000100040000L});
public static final BitSet FOLLOW_AT_RPAREN_in_arguments1253 = new BitSet(new long[]{0x0000000000000002L}); public static final BitSet FOLLOW_AT_RPAREN_in_arguments1253 = new BitSet(new long[]{0x0000000000000002L});
} }

127
lucene/expressions/src/java/org/apache/lucene/expressions/js/VariableContext.java Normal file
View File

@ -0,0 +1,127 @@
package org.apache.lucene.expressions.js;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.List;
/**
* A helper to parse the context of a variable name, which is the base variable, followed by the
* sequence of array (integer or string indexed) and member accesses.
*/
public class VariableContext {
/**
* Represents what a piece of a variable does.
*/
public static enum Type {
/**
* A member of the previous context (ie "dot" access).
*/
MEMBER,
/**
* Brackets containing a string as the "index".
*/
STR_INDEX,
/**
* Brackets containg an integer index (ie an array).
*/
INT_INDEX
}
/**
* The type of this piece of a variable.
*/
public final Type type;
/**
* The text of this piece of the variable. Used for {@link Type#MEMBER} and {@link Type#STR_INDEX} types.
*/
public final String text;
/**
* The integer value for this piece of the variable. Used for {@link Type#INT_INDEX}.
*/
public final int integer;
private VariableContext(Type c, String s, int i) {
type = c;
text = s;
integer = i;
}
/**
* Parses a normalized javascript variable. All strings in the variable should be single quoted,
* and no spaces (except possibly within strings).
*/
public static final VariableContext[] parse(String variable) {
char[] text = variable.toCharArray();
List<VariableContext> contexts = new ArrayList<>();
int i = addMember(text, 0, contexts); // base variable is a "member" of the global namespace
while (i < text.length) {
if (text[i] == '[') {
if (text[++i] == '\'') {
i = addStringIndex(text, i, contexts);
} else {
i = addIntIndex(text, i, contexts);
}
++i; // move past end bracket
} else { // text[i] == '.', ie object member
i = addMember(text, i + 1, contexts);
}
}
return contexts.toArray(new VariableContext[contexts.size()]);
}
// i points to start of member name
private static int addMember(final char[] text, int i, List<VariableContext> contexts) {
int j = i + 1;
while (j < text.length && text[j] != '[' && text[j] != '.') ++j; // find first array or member access
contexts.add(new VariableContext(Type.MEMBER, new String(text, i, j - i), -1));
return j;
}
// i points to start of single quoted index
private static int addStringIndex(final char[] text, int i, List<VariableContext> contexts) {
++i; // move past quote
int j = i;
while (text[j] != '\'') { // find end of single quoted string
if (text[j] == '\\') ++j; // skip over escapes
++j;
}
StringBuffer buf = new StringBuffer(j - i); // space for string, without end quote
while (i < j) { // copy string to buffer (without begin/end quotes)
if (text[i] == '\\') ++i; // unescape escapes
buf.append(text[i]);
++i;
}
contexts.add(new VariableContext(Type.STR_INDEX, buf.toString(), -1));
return j + 1; // move past quote, return end bracket location
}
// i points to start of integer index
private static int addIntIndex(final char[] text, int i, List<VariableContext> contexts) {
int j = i + 1;
while (text[j] != ']') ++j; // find end of array access
int index = Integer.parseInt(new String(text, i, j - i));
contexts.add(new VariableContext(Type.INT_INDEX, null, index));
return j ;
}
}

75
lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java
View File

@ -4,9 +4,13 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.expressions.js.JavascriptCompiler; import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.apache.lucene.expressions.js.VariableContext;
import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSearcher;
@ -19,6 +23,10 @@ import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import static org.apache.lucene.expressions.js.VariableContext.Type.MEMBER;
import static org.apache.lucene.expressions.js.VariableContext.Type.STR_INDEX;
import static org.apache.lucene.expressions.js.VariableContext.Type.INT_INDEX;
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
@ -224,4 +232,71 @@ public class TestDemoExpressions extends LuceneTestCase {
d = (FieldDoc) td.scoreDocs[2]; d = (FieldDoc) td.scoreDocs[2];
assertEquals(5.2842D, (Double)d.fields[0], 1E-4); assertEquals(5.2842D, (Double)d.fields[0], 1E-4);
} }
public void testStaticExtendedVariableExample() throws Exception {
Expression popularity = JavascriptCompiler.compile("doc[\"popularity\"].value");
SimpleBindings bindings = new SimpleBindings();
bindings.add("doc['popularity'].value", new IntFieldSource("popularity"));
Sort sort = new Sort(popularity.getSortField(bindings, true));
TopFieldDocs td = searcher.search(new MatchAllDocsQuery(), null, 3, sort);
FieldDoc d = (FieldDoc)td.scoreDocs[0];
assertEquals(20D, (Double)d.fields[0], 1E-4);
d = (FieldDoc)td.scoreDocs[1];
assertEquals(5D, (Double)d.fields[0], 1E-4);
d = (FieldDoc)td.scoreDocs[2];
assertEquals(2D, (Double)d.fields[0], 1E-4);
}
public void testDynamicExtendedVariableExample() throws Exception {
Expression popularity = JavascriptCompiler.compile("doc['popularity'].value + magicarray[0] + fourtytwo");
// The following is an example of how to write bindings which parse the variable name into pieces.
// Note, however, that this requires a lot of error checking. Each "error case" below should be
// filled in with proper error messages for a real use case.
Bindings bindings = new Bindings() {
@Override
public ValueSource getValueSource(String name) {
VariableContext[] var = VariableContext.parse(name);
assert var[0].type == MEMBER;
String base = var[0].text;
if (base.equals("doc")) {
if (var.length > 1 && var[1].type == STR_INDEX) {
String field = var[1].text;
if (var.length > 2 && var[2].type == MEMBER && var[2].text.equals("value")) {
return new IntFieldSource(field);
} else {
fail("member: " + var[2].text);// error case, non/missing "value" member access
}
} else {
fail();// error case, doc should be a str indexed array
}
} else if (base.equals("magicarray")) {
if (var.length > 1 && var[1].type == INT_INDEX) {
return new DoubleConstValueSource(2048);
} else {
fail();// error case, magic array isn't an array
}
} else if (base.equals("fourtytwo")) {
return new DoubleConstValueSource(42);
} else {
fail();// error case (variable doesn't exist)
}
throw new IllegalArgumentException("Illegal reference '" + name + "'");
}
};
Sort sort = new Sort(popularity.getSortField(bindings, false));
TopFieldDocs td = searcher.search(new MatchAllDocsQuery(), null, 3, sort);
FieldDoc d = (FieldDoc)td.scoreDocs[0];
assertEquals(2092D, (Double)d.fields[0], 1E-4);
d = (FieldDoc)td.scoreDocs[1];
assertEquals(2095D, (Double)d.fields[0], 1E-4);
d = (FieldDoc)td.scoreDocs[2];
assertEquals(2110D, (Double)d.fields[0], 1E-4);
}
} }

102
lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.expressions.js;
import java.text.ParseException; import java.text.ParseException;
import org.apache.lucene.expressions.Expression;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
public class TestJavascriptCompiler extends LuceneTestCase { public class TestJavascriptCompiler extends LuceneTestCase {
@ -29,39 +30,60 @@ public class TestJavascriptCompiler extends LuceneTestCase {
assertNotNull(JavascriptCompiler.compile("logn(2, 20+10-5.0)")); assertNotNull(JavascriptCompiler.compile("logn(2, 20+10-5.0)"));
} }
public void testValidNamespaces() throws Exception { public void testValidVariables() throws Exception {
assertNotNull(JavascriptCompiler.compile("object.valid0")); doTestValidVariable("object.valid0");
assertNotNull(JavascriptCompiler.compile("object0.object1.valid1")); doTestValidVariable("object0.object1.valid1");
doTestValidVariable("array0[1]");
doTestValidVariable("array0[1].x");
doTestValidVariable("multiarray[0][0]");
doTestValidVariable("multiarray[0][0].x");
doTestValidVariable("strindex['hello']");
doTestValidVariable("strindex[\"hello\"]", "strindex['hello']");
doTestValidVariable("empty['']");
doTestValidVariable("empty[\"\"]", "empty['']");
doTestValidVariable("strindex['\u304A\u65E9\u3046\u3054\u3056\u3044\u307E\u3059']");
doTestValidVariable("strindex[\"\u304A\u65E9\u3046\u3054\u3056\u3044\u307E\u3059\"]",
"strindex['\u304A\u65E9\u3046\u3054\u3056\u3044\u307E\u3059']");
doTestValidVariable("escapes['\\\\\\'']");
doTestValidVariable("escapes[\"\\\\\\\"\"]", "escapes['\\\\\"']");
doTestValidVariable("mixed[23]['key'].sub.sub");
doTestValidVariable("mixed[23]['key'].sub.sub[1]");
doTestValidVariable("mixed[23]['key'].sub.sub[1].sub");
doTestValidVariable("mixed[23]['key'].sub.sub[1].sub['abc']");
} }
public void testInvalidNamespaces() throws Exception { void doTestValidVariable(String variable) throws Exception {
try { doTestValidVariable(variable, variable);
JavascriptCompiler.compile("object.0invalid"); }
fail();
}
catch (ParseException expected) {
//expected
}
try { void doTestValidVariable(String variable, String output) throws Exception {
JavascriptCompiler.compile("0.invalid"); Expression e = JavascriptCompiler.compile(variable);
fail(); assertNotNull(e);
} assertEquals(1, e.variables.length);
catch (ParseException expected) { assertEquals(output, e.variables[0]);
//expected }
}
try { public void testInvalidVariables() throws Exception {
JavascriptCompiler.compile("object..invalid"); doTestInvalidVariable("object.0invalid");
fail(); doTestInvalidVariable("0.invalid");
} doTestInvalidVariable("object..invalid");
catch (ParseException expected) { doTestInvalidVariable(".invalid");
//expected doTestInvalidVariable("negative[-1]");
} doTestInvalidVariable("float[1.0]");
doTestInvalidVariable("missing_end['abc]");
doTestInvalidVariable("missing_end[\"abc]");
doTestInvalidVariable("missing_begin[abc']");
doTestInvalidVariable("missing_begin[abc\"]");
doTestInvalidVariable("dot_needed[1]sub");
doTestInvalidVariable("dot_needed[1]sub");
doTestInvalidVariable("opposite_escape['\\\"']");
doTestInvalidVariable("opposite_escape[\"\\'\"]");
}
void doTestInvalidVariable(String variable) {
try { try {
JavascriptCompiler.compile(".invalid"); JavascriptCompiler.compile(variable);
fail(); fail("\"" + variable + " should have failed to compile");
} }
catch (ParseException expected) { catch (ParseException expected) {
//expected //expected
@ -152,4 +174,30 @@ public class TestJavascriptCompiler extends LuceneTestCase {
assertTrue(expected.getMessage().contains("arguments for method call")); assertTrue(expected.getMessage().contains("arguments for method call"));
} }
} }
public void testVariableNormalization() throws Exception {
// multiple double quotes
Expression x = JavascriptCompiler.compile("foo[\"a\"][\"b\"]");
assertEquals("foo['a']['b']", x.variables[0]);
// single and double in the same var
x = JavascriptCompiler.compile("foo['a'][\"b\"]");
assertEquals("foo['a']['b']", x.variables[0]);
// escapes remain the same in single quoted strings
x = JavascriptCompiler.compile("foo['\\\\\\'\"']");
assertEquals("foo['\\\\\\'\"']", x.variables[0]);
// single quotes are escaped
x = JavascriptCompiler.compile("foo[\"'\"]");
assertEquals("foo['\\'']", x.variables[0]);
// double quotes are unescaped
x = JavascriptCompiler.compile("foo[\"\\\"\"]");
assertEquals("foo['\"']", x.variables[0]);
// backslash escapes are kept the same
x = JavascriptCompiler.compile("foo['\\\\'][\"\\\\\"]");
assertEquals("foo['\\\\']['\\\\']", x.variables[0]);
}
} }

69
lucene/expressions/src/test/org/apache/lucene/expressions/js/TestVariableContext.java Normal file
View File

@ -0,0 +1,69 @@
package org.apache.lucene.expressions.js;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.util.LuceneTestCase;
import static org.apache.lucene.expressions.js.VariableContext.Type.MEMBER;
import static org.apache.lucene.expressions.js.VariableContext.Type.STR_INDEX;
import static org.apache.lucene.expressions.js.VariableContext.Type.INT_INDEX;
public class TestVariableContext extends LuceneTestCase {
public void testSimpleVar() {
VariableContext[] x = VariableContext.parse("foo");
assertEquals(1, x.length);
assertEquals(x[0].type, MEMBER);
assertEquals(x[0].text, "foo");
}
public void testEmptyString() {
VariableContext[] x = VariableContext.parse("foo['']");
assertEquals(2, x.length);
assertEquals(x[1].type, STR_INDEX);
assertEquals(x[1].text, "");
}
public void testUnescapeString() {
VariableContext[] x = VariableContext.parse("foo['\\'\\\\']");
assertEquals(2, x.length);
assertEquals(x[1].type, STR_INDEX);
assertEquals(x[1].text, "'\\");
}
public void testMember() {
VariableContext[] x = VariableContext.parse("foo.bar");
assertEquals(2, x.length);
assertEquals(x[1].type, MEMBER);
assertEquals(x[1].text, "bar");
}
public void testMemberFollowedByMember() {
VariableContext[] x = VariableContext.parse("foo.bar.baz");
assertEquals(3, x.length);
assertEquals(x[2].type, MEMBER);
assertEquals(x[2].text, "baz");
}
public void testMemberFollowedByIntArray() {
VariableContext[] x = VariableContext.parse("foo.bar[1]");
assertEquals(3, x.length);
assertEquals(x[2].type, INT_INDEX);
assertEquals(x[2].integer, 1);
}
}