[OLINGO-568] More tests and fixes
This commit is contained in:
mibo
parent
233ea61f3b
commit
63db8b36c4
|
|
@ -129,6 +129,9 @@ public class SearchParser {
|
|||
SearchExpression se = left;
|
||||
if (isTerm()) {
|
||||
se = processTerm();
|
||||
if(isTerm()) {
|
||||
se = processAnd(se);
|
||||
}
|
||||
se = new SearchBinaryImpl(left, SearchBinaryOperatorKind.AND, se);
|
||||
return processSearchExpression(se);
|
||||
} else {
|
||||
|
|
@ -154,6 +157,10 @@ public class SearchParser {
|
|||
if (isToken(Token.WORD) || isToken(Token.PHRASE)) {
|
||||
return new SearchUnaryImpl(processWordOrPhrase());
|
||||
}
|
||||
if(isEof()) {
|
||||
throw new SearchParserException("NOT must be followed by a term.",
|
||||
SearchParserException.MessageKeys.INVALID_NOT_OPERAND, "EOF");
|
||||
}
|
||||
throw new SearchParserException("NOT must be followed by a term not a " + token.getToken(),
|
||||
SearchParserException.MessageKeys.INVALID_NOT_OPERAND, token.getToken().toString());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ public class SearchTokenizer {
|
|||
}
|
||||
|
||||
public State invalid() throws SearchTokenizerException {
|
||||
throw new SearchTokenizerException("Token " + this.getToken() + " is in invalid state ",
|
||||
throw new SearchTokenizerException("Token " + this.getToken() + " is in invalid state.",
|
||||
SearchTokenizerException.MessageKeys.INVALID_TOKEN_STATE);
|
||||
}
|
||||
|
||||
|
|
@ -428,7 +428,7 @@ public class SearchTokenizer {
|
|||
if(closed) {
|
||||
return finish();
|
||||
}
|
||||
return super.close();
|
||||
return invalid();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -475,15 +475,18 @@ public class SearchTokenizer {
|
|||
} else if (literal.length() == 3 && isWhitespace(c)) {
|
||||
finish();
|
||||
return new BeforePhraseOrWordRwsState();
|
||||
} else if(isWhitespace(c)) {
|
||||
changeToken(Token.WORD).finish();
|
||||
return new RwsState();
|
||||
}
|
||||
return forbidden(c);
|
||||
return new SearchWordState(this).nextChar(c);
|
||||
}
|
||||
@Override
|
||||
public State close() throws SearchTokenizerException {
|
||||
if(Token.NOT.name().equals(literal.toString())) {
|
||||
return finish();
|
||||
}
|
||||
return super.close();
|
||||
return changeToken(Token.WORD).finish();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -504,9 +507,18 @@ public class SearchTokenizer {
|
|||
} else if (literal.length() == 3 && isWhitespace(c)) {
|
||||
finish();
|
||||
return new BeforeSearchExpressionRwsState();
|
||||
} else {
|
||||
return new SearchWordState(this);
|
||||
} else if(isWhitespace(c)) {
|
||||
changeToken(Token.WORD).finish();
|
||||
return new RwsState();
|
||||
}
|
||||
return new SearchWordState(this).nextChar(c);
|
||||
}
|
||||
@Override
|
||||
public State close() throws SearchTokenizerException {
|
||||
if(Token.AND.name().equals(literal.toString())) {
|
||||
return finish();
|
||||
}
|
||||
return changeToken(Token.WORD).finish();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -517,7 +529,6 @@ public class SearchTokenizer {
|
|||
forbidden(c);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public State nextChar(char c) throws SearchTokenizerException {
|
||||
if (literal.length() == 1 && (c == CHAR_R)) {
|
||||
|
|
@ -525,9 +536,18 @@ public class SearchTokenizer {
|
|||
} else if (literal.length() == 2 && isWhitespace(c)) {
|
||||
finish();
|
||||
return new BeforeSearchExpressionRwsState();
|
||||
} else {
|
||||
return new SearchWordState(this);
|
||||
} else if(isWhitespace(c)) {
|
||||
changeToken(Token.WORD).finish();
|
||||
return new RwsState();
|
||||
}
|
||||
return new SearchWordState(this).nextChar(c);
|
||||
}
|
||||
@Override
|
||||
public State close() throws SearchTokenizerException {
|
||||
if(Token.OR.name().equals(literal.toString())) {
|
||||
return finish();
|
||||
}
|
||||
return changeToken(Token.WORD).finish();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,15 +18,9 @@
|
|||
*/
|
||||
package org.apache.olingo.server.core.uri.parser.search;
|
||||
|
||||
import static org.apache.olingo.server.api.uri.queryoption.search.SearchBinaryOperatorKind.AND;
|
||||
import static org.apache.olingo.server.api.uri.queryoption.search.SearchBinaryOperatorKind.OR;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
|
||||
import org.apache.olingo.server.api.ODataLibraryException;
|
||||
import org.apache.olingo.server.api.uri.queryoption.SearchOption;
|
||||
import org.apache.olingo.server.api.uri.queryoption.search.SearchExpression;
|
||||
import org.apache.olingo.server.api.uri.queryoption.search.SearchUnary;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
|
|
@ -34,149 +28,86 @@ public class SearchParserAndTokenizerTest {
|
|||
|
||||
@Test
|
||||
public void basicParsing() throws Exception {
|
||||
SearchExpressionValidator.init("\"99\"")
|
||||
.validate(with("99"));
|
||||
SearchExpressionValidator.init("a")
|
||||
.validate(with("a"));
|
||||
SearchExpressionValidator.init("a AND b")
|
||||
.validate(with("a", and("b")));
|
||||
SearchExpressionValidator.init("a AND b AND c")
|
||||
.validate("{{'a' AND 'b'} AND 'c'}");
|
||||
SearchExpressionValidator.init("a OR b")
|
||||
.validate(with("a", or("b")));
|
||||
SearchExpressionValidator.init("a OR b OR c")
|
||||
.validate(with("a", or("b", or("c"))));
|
||||
assertQuery("\"99\"").resultsIn("'99'");
|
||||
assertQuery("a").resultsIn("'a'");
|
||||
assertQuery("a AND b").resultsIn("{'a' AND 'b'}");
|
||||
assertQuery("a AND b AND c").resultsIn("{{'a' AND 'b'} AND 'c'}");
|
||||
assertQuery("a OR b").resultsIn("{'a' OR 'b'}");
|
||||
assertQuery("a OR b OR c").resultsIn("{'a' OR {'b' OR 'c'}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mixedParsing() throws Exception {
|
||||
SearchExpressionValidator.init("a AND b OR c")
|
||||
.validate("{{'a' AND 'b'} OR 'c'}");
|
||||
SearchExpressionValidator.init("a OR b AND c")
|
||||
.validate("{'a' OR {'b' AND 'c'}}");
|
||||
assertQuery("a AND b OR c").resultsIn("{{'a' AND 'b'} OR 'c'}");
|
||||
assertQuery("a OR b AND c").resultsIn("{'a' OR {'b' AND 'c'}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void notParsing() throws Exception {
|
||||
SearchExpressionValidator.init("NOT a AND b OR c")
|
||||
.validate("{{{NOT 'a'} AND 'b'} OR 'c'}");
|
||||
SearchExpressionValidator.init("a OR b AND NOT c")
|
||||
.validate("{'a' OR {'b' AND {NOT 'c'}}}");
|
||||
assertQuery("NOT a AND b OR c").resultsIn("{{{NOT 'a'} AND 'b'} OR 'c'}");
|
||||
assertQuery("a OR b AND NOT c").resultsIn("{'a' OR {'b' AND {NOT 'c'}}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parenthesesParsing() throws Exception {
|
||||
SearchExpressionValidator.init("a AND (b OR c)")
|
||||
.validate("{'a' AND {'b' OR 'c'}}");
|
||||
SearchExpressionValidator.init("(a OR b) AND NOT c")
|
||||
.validate("{{'a' OR 'b'} AND {NOT 'c'}}");
|
||||
assertQuery("a AND (b OR c)").resultsIn("{'a' AND {'b' OR 'c'}}");
|
||||
assertQuery("(a OR b) AND NOT c").resultsIn("{{'a' OR 'b'} AND {NOT 'c'}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseImplicitAnd() throws Exception {
|
||||
assertQuery("a b").resultsIn("{'a' AND 'b'}");
|
||||
assertQuery("a b c").resultsIn("{'a' AND {'b' AND 'c'}}");
|
||||
assertQuery("a and b").resultsIn("{'a' AND {'and' AND 'b'}}");
|
||||
assertQuery("a b OR c").resultsIn("{{'a' AND 'b'} OR 'c'}");
|
||||
assertQuery("a \"bc123\" OR c").resultsIn("{{'a' AND 'bc123'} OR 'c'}");
|
||||
assertQuery("(a OR x) bc c").resultsIn("{{'a' OR 'x'} AND {'bc' AND 'c'}}");
|
||||
assertQuery("one ((a OR x) bc c)").resultsIn("{'one' AND {{'a' OR 'x'} AND {'bc' AND 'c'}}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void invalidSearchQuery() throws Exception {
|
||||
SearchExpressionValidator.init("99").validate(SearchParserException.class,
|
||||
SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
|
||||
assertQuery("99").resultsIn(SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
|
||||
assertQuery("NOT").resultsIn(SearchParserException.MessageKeys.INVALID_NOT_OPERAND);
|
||||
assertQuery("AND").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION);
|
||||
assertQuery("OR").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION);
|
||||
}
|
||||
|
||||
private static SearchExpression with(String term) {
|
||||
return new SearchTermImpl(term);
|
||||
private static Validator assertQuery(String searchQuery) {
|
||||
return Validator.init(searchQuery);
|
||||
}
|
||||
|
||||
private static SearchExpression with(String left, SearchExpression right) {
|
||||
setLeftField(left, right);
|
||||
return right;
|
||||
}
|
||||
|
||||
private static SearchUnary with(SearchUnary unary) {
|
||||
return unary;
|
||||
}
|
||||
|
||||
private static SearchExpression or(String left, SearchExpression right) {
|
||||
SearchExpression or = or(right);
|
||||
setLeftField(left, right);
|
||||
return or;
|
||||
}
|
||||
|
||||
private static SearchExpression and(String left, SearchExpression right) {
|
||||
SearchExpression and = and(right);
|
||||
setLeftField(left, right);
|
||||
return and;
|
||||
}
|
||||
|
||||
private static SearchExpression or(SearchExpression right) {
|
||||
return new SearchBinaryImpl(null, OR, right);
|
||||
}
|
||||
|
||||
private static SearchExpression and(SearchExpression right) {
|
||||
return new SearchBinaryImpl(null, AND, right);
|
||||
}
|
||||
|
||||
private static SearchExpression and(String right) {
|
||||
return and(new SearchTermImpl(right));
|
||||
}
|
||||
|
||||
private static SearchExpression or(String right) {
|
||||
return or(new SearchTermImpl(right));
|
||||
}
|
||||
|
||||
private static SearchUnary not(String term) {
|
||||
return new SearchUnaryImpl(new SearchTermImpl(term));
|
||||
}
|
||||
|
||||
private static void setLeftField(String left, SearchExpression se) {
|
||||
try {
|
||||
Field field = null;
|
||||
if (se instanceof SearchUnaryImpl) {
|
||||
field = SearchBinaryImpl.class.getDeclaredField("operand");
|
||||
} else if (se instanceof SearchBinaryImpl) {
|
||||
field = SearchBinaryImpl.class.getDeclaredField("left");
|
||||
} else {
|
||||
Assert.fail("Unexpected exception: " + se.getClass());
|
||||
}
|
||||
field.setAccessible(true);
|
||||
field.set(se, new SearchTermImpl(left));
|
||||
} catch (Exception e) {
|
||||
Assert.fail("Unexpected exception: " + e.getClass());
|
||||
}
|
||||
}
|
||||
|
||||
private static class SearchExpressionValidator {
|
||||
private static class Validator {
|
||||
private boolean log;
|
||||
private final String searchQuery;
|
||||
|
||||
private SearchExpressionValidator(String searchQuery) {
|
||||
private Validator(String searchQuery) {
|
||||
this.searchQuery = searchQuery;
|
||||
}
|
||||
|
||||
private static SearchExpressionValidator init(String searchQuery) {
|
||||
return new SearchExpressionValidator(searchQuery);
|
||||
private static Validator init(String searchQuery) {
|
||||
return new Validator(searchQuery);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private SearchExpressionValidator enableLogging() {
|
||||
private Validator withLogging() {
|
||||
log = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
private void validate(Class<? extends ODataLibraryException> exception, ODataLibraryException.MessageKey key)
|
||||
private void resultsIn(ODataLibraryException.MessageKey key)
|
||||
throws SearchTokenizerException {
|
||||
try {
|
||||
validate(searchQuery);
|
||||
resultsIn(searchQuery);
|
||||
} catch (ODataLibraryException e) {
|
||||
Assert.assertEquals(exception, e.getClass());
|
||||
Assert.assertEquals(SearchParserException.class, e.getClass());
|
||||
Assert.assertEquals(key, e.getMessageKey());
|
||||
return;
|
||||
}
|
||||
Assert.fail("Expected exception " + exception.getClass().getSimpleName() + " was not thrown.");
|
||||
Assert.fail("SearchParserException with message key " + key.getKey() + " was not thrown.");
|
||||
}
|
||||
|
||||
private void validate(SearchExpression expectedSearchExpression) throws SearchTokenizerException,
|
||||
SearchParserException {
|
||||
final SearchExpression searchExpression = getSearchExpression();
|
||||
Assert.assertEquals(expectedSearchExpression.toString(), searchExpression.toString());
|
||||
}
|
||||
|
||||
private void validate(String expectedSearchExpression) throws SearchTokenizerException, SearchParserException {
|
||||
private void resultsIn(String expectedSearchExpression) throws SearchTokenizerException, SearchParserException {
|
||||
final SearchExpression searchExpression = getSearchExpression();
|
||||
Assert.assertEquals(expectedSearchExpression, searchExpression.toString());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ public class SearchParserTest extends SearchParser {
|
|||
se = run(Token.PHRASE);
|
||||
assertEquals("'phrase1'", se.toString());
|
||||
assertTrue(se.isSearchTerm());
|
||||
// TODO: Check if quotation marks should be part of the string we deliver
|
||||
assertEquals("phrase1", se.asSearchTerm().getSearchTerm());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -38,52 +38,28 @@ public class SearchTokenizerTest {
|
|||
|
||||
@Test
|
||||
public void parseBasics() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
|
||||
//
|
||||
result = tokenizer.tokenize("abc");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
|
||||
result = tokenizer.tokenize("NOT abc");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(NOT, result.get(0).getToken());
|
||||
Assert.assertEquals(WORD, result.get(1).getToken());
|
||||
|
||||
result = tokenizer.tokenize("(abc)");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(OPEN, result.get(0).getToken());
|
||||
Assert.assertEquals(WORD, result.get(1).getToken());
|
||||
Assert.assertEquals(CLOSE, result.get(2).getToken());
|
||||
|
||||
result = tokenizer.tokenize("((abc))");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(OPEN, result.get(0).getToken());
|
||||
Assert.assertEquals(WORD, result.get(2).getToken());
|
||||
Assert.assertEquals(CLOSE, result.get(4).getToken());
|
||||
assertQuery("abd").resultsIn(WORD);
|
||||
assertQuery("NOT abc").resultsIn(NOT, WORD);
|
||||
assertQuery("(abc)").resultsIn(OPEN, WORD, CLOSE);
|
||||
assertQuery("((abc))").resultsIn(OPEN, OPEN, WORD, CLOSE, CLOSE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseWords() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
assertQuery("somesimpleword").resultsIn(WORD);
|
||||
assertQuery("anotherWord\u1234").resultsIn(WORD);
|
||||
// special
|
||||
assertQuery("NO").resultsIn(word("NO"));
|
||||
assertQuery("N").resultsIn(word("N"));
|
||||
assertQuery("A").resultsIn(word("A"));
|
||||
assertQuery("AN").resultsIn(word("AN"));
|
||||
assertQuery("O").resultsIn(word("O"));
|
||||
// invalid
|
||||
assertQuery("notAw0rd").resultsIn(SearchTokenizerException.MessageKeys.FORBIDDEN_CHARACTER);
|
||||
}
|
||||
|
||||
//
|
||||
result = tokenizer.tokenize("abc");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
|
||||
//
|
||||
result = tokenizer.tokenize("anotherWord\u1234");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
private Validator.Tuple word(String literal) {
|
||||
return Validator.tuple(WORD, literal);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -91,7 +67,7 @@ public class SearchTokenizerTest {
|
|||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
|
||||
TokenizerValidator.init("abc AND \"x-y_z\" AND olingo").validate();
|
||||
assertQuery("abc AND \"x-y_z\" AND olingo");
|
||||
|
||||
//
|
||||
result = tokenizer.tokenize("\"abc\"");
|
||||
|
|
@ -113,7 +89,7 @@ public class SearchTokenizerTest {
|
|||
Assert.assertEquals(PHRASE, result.get(0).getToken());
|
||||
Assert.assertEquals("\"99_88.\"", result.get(0).getLiteral());
|
||||
|
||||
TokenizerValidator.init("abc or \"xyz\"").validate(WORD, WORD, PHRASE);
|
||||
assertQuery("abc or \"xyz\"").resultsIn(WORD, WORD, PHRASE);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -124,165 +100,95 @@ public class SearchTokenizerTest {
|
|||
@Ignore("Test must be moved to SearchParserTest and SearchParserAndTokenizerTest")
|
||||
public void parsePhraseAbnfTestcases() throws Exception {
|
||||
// <TestCase Name="5.1.7 Search - simple phrase" Rule="queryOptions">
|
||||
TokenizerValidator.init("\"blue%20green\"").validate();
|
||||
assertQuery("\"blue%20green\"");
|
||||
// <TestCase Name="5.1.7 Search - simple phrase" Rule="queryOptions">
|
||||
TokenizerValidator.init("\"blue%20green%22").validate();
|
||||
assertQuery("\"blue%20green%22");
|
||||
// <TestCase Name="5.1.7 Search - phrase with escaped double-quote" Rule="queryOptions">
|
||||
// <Input>$search="blue\"green"</Input>
|
||||
TokenizerValidator.init("\"blue\\\"green\"").validate();
|
||||
assertQuery("\"blue\\\"green\"");
|
||||
|
||||
// <TestCase Name="5.1.7 Search - phrase with escaped backslash" Rule="queryOptions">
|
||||
// <Input>$search="blue\\green"</Input>
|
||||
TokenizerValidator.init("\"blue\\\\green\"").validate();
|
||||
assertQuery("\"blue\\\\green\"");
|
||||
|
||||
// <TestCase Name="5.1.7 Search - phrase with unescaped double-quote" Rule="queryOptions" FailAt="14">
|
||||
TokenizerValidator.init("\"blue\"green\"").validate();
|
||||
assertQuery("\"blue\"green\"");
|
||||
|
||||
// <TestCase Name="5.1.7 Search - phrase with unescaped double-quote" Rule="queryOptions" FailAt="16">
|
||||
TokenizerValidator.init("\"blue%22green\"").validate();
|
||||
assertQuery("\"blue%22green\"");
|
||||
|
||||
// <TestCase Name="5.1.7 Search - implicit AND" Rule="queryOptions">
|
||||
// <Input>$search=blue green</Input>
|
||||
// SearchValidator.init("\"blue%20green\"").validate();
|
||||
// SearchassertQuery("\"blue%20green\"").resultsIn();
|
||||
// <TestCase Name="5.1.7 Search - implicit AND, encoced" Rule="queryOptions">
|
||||
// SearchValidator.init("blue%20green").validate();
|
||||
// SearchassertQuery("blue%20green").resultsIn();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void parseNot() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
|
||||
result = tokenizer.tokenize("NOT abc");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(NOT, result.get(0).getToken());
|
||||
Assert.assertEquals(WORD, result.get(1).getToken());
|
||||
|
||||
TokenizerValidator.init("not abc").addExpected(WORD, WORD).validate();
|
||||
TokenizerValidator.init("NOT abc").addExpected(NOT, WORD).validate();
|
||||
TokenizerValidator.init("NOT \"abc\"").addExpected(NOT, PHRASE).validate();
|
||||
TokenizerValidator.init("NOT (sdf)").validate(SearchTokenizerException.class);
|
||||
assertQuery("NOT").resultsIn(NOT);
|
||||
assertQuery(" NOT ").resultsIn(NOT);
|
||||
assertQuery("NOT abc").resultsIn(NOT, WORD);
|
||||
assertQuery("not abc").resultsIn(WORD, WORD);
|
||||
assertQuery("NOT abc").resultsIn(NOT, WORD);
|
||||
assertQuery("NOT \"abc\"").resultsIn(NOT, PHRASE);
|
||||
assertQuery("NObody").resultsIn(WORD);
|
||||
assertQuery("Nobody").resultsIn(WORD);
|
||||
assertQuery("NOT (sdf)").resultsIn(SearchTokenizerException.MessageKeys.FORBIDDEN_CHARACTER);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseOr() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
|
||||
result = tokenizer.tokenize("abc OR xyz");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(OR, result.get(1).getToken());
|
||||
Assert.assertEquals(WORD, result.get(2).getToken());
|
||||
|
||||
result = tokenizer.tokenize("abc OR xyz OR olingo");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(OR, result.get(1).getToken());
|
||||
Assert.assertEquals(WORD, result.get(2).getToken());
|
||||
Assert.assertEquals(OR, result.get(3).getToken());
|
||||
Assert.assertEquals(WORD, result.get(4).getToken());
|
||||
|
||||
TokenizerValidator.init("abc or xyz").addExpected(WORD, WORD, WORD).validate();
|
||||
assertQuery("OR").resultsIn(OR);
|
||||
assertQuery(" OR ").resultsIn(OR);
|
||||
assertQuery("OR xyz").resultsIn(OR, WORD);
|
||||
assertQuery("abc OR xyz").resultsIn(WORD, OR, WORD);
|
||||
assertQuery("abc OR xyz OR olingo").resultsIn(WORD, OR, WORD, OR, WORD);
|
||||
assertQuery("abc or xyz").addExpected(WORD, WORD, WORD);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseImplicitAnd() throws SearchTokenizerException {
|
||||
TokenizerValidator.init("a b").addExpected(WORD, WORD).validate();
|
||||
TokenizerValidator.init("a b OR c").addExpected(WORD, WORD, OR, WORD).validate();
|
||||
TokenizerValidator.init("a bc OR c").addExpected(WORD, WORD, OR, WORD).validate();
|
||||
TokenizerValidator.init("a bc c").addExpected(WORD, WORD, WORD).validate();
|
||||
TokenizerValidator.init("(a OR x) bc c").addExpected(OPEN, WORD, OR, WORD, CLOSE, WORD, WORD).validate();
|
||||
assertQuery("a b").resultsIn(WORD, WORD);
|
||||
assertQuery("a b OR c").resultsIn(WORD, WORD, OR, WORD);
|
||||
assertQuery("a bc OR c").resultsIn(WORD, WORD, OR, WORD);
|
||||
assertQuery("a bc c").resultsIn(WORD, WORD, WORD);
|
||||
assertQuery("(a OR x) bc c").resultsIn(OPEN, WORD, OR, WORD, CLOSE, WORD, WORD);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseAnd() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
|
||||
result = tokenizer.tokenize("abc AND xyz");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(AND, result.get(1).getToken());
|
||||
Assert.assertEquals(WORD, result.get(2).getToken());
|
||||
assertQuery("AND").resultsIn(AND);
|
||||
assertQuery(" AND ").resultsIn(AND);
|
||||
|
||||
assertQuery("abc AND xyz").resultsIn(WORD, AND, WORD);
|
||||
// no lower case allowed for AND
|
||||
result = tokenizer.tokenize("abc and xyz");
|
||||
Assert.assertNotNull(result);
|
||||
Assert.assertEquals(3, result.size());
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(WORD, result.get(1).getToken());
|
||||
Assert.assertEquals(WORD, result.get(2).getToken());
|
||||
|
||||
// implicit AND
|
||||
result = tokenizer.tokenize("abc xyz");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(WORD, result.get(1).getToken());
|
||||
|
||||
result = tokenizer.tokenize("abc AND xyz AND olingo");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(AND, result.get(1).getToken());
|
||||
Assert.assertEquals(WORD, result.get(2).getToken());
|
||||
Assert.assertEquals(AND, result.get(3).getToken());
|
||||
Assert.assertEquals(WORD, result.get(4).getToken());
|
||||
|
||||
result = tokenizer.tokenize("abc AND \"x-y_z\" AND olingo");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(AND, result.get(1).getToken());
|
||||
Assert.assertEquals(PHRASE, result.get(2).getToken());
|
||||
Assert.assertEquals("\"x-y_z\"", result.get(2).getLiteral());
|
||||
Assert.assertEquals(AND, result.get(3).getToken());
|
||||
Assert.assertEquals(WORD, result.get(4).getToken());
|
||||
assertQuery("abc and xyz").resultsIn(WORD, WORD, WORD);
|
||||
// implicit AND is handled by parser (and not tokenizer)
|
||||
assertQuery("abc xyz").resultsIn(WORD, WORD);
|
||||
assertQuery("abc AND xyz AND olingo").resultsIn(WORD, AND, WORD, AND, WORD);
|
||||
assertQuery("abc AND \"x-y_z\" AND olingo")
|
||||
.resultsIn(WORD, AND, PHRASE, AND, WORD);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseAndOr() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
|
||||
result = tokenizer.tokenize("abc AND xyz OR olingo");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Assert.assertEquals(WORD, result.get(0).getToken());
|
||||
Assert.assertEquals(AND, result.get(1).getToken());
|
||||
Assert.assertEquals(WORD, result.get(2).getToken());
|
||||
Assert.assertEquals(OR, result.get(3).getToken());
|
||||
Assert.assertEquals(WORD, result.get(4).getToken());
|
||||
|
||||
TokenizerValidator.init("abc AND ANDsomething")
|
||||
.addExpected(WORD, AND, WORD).validate();
|
||||
assertQuery("OR AND ").resultsIn(OR, AND);
|
||||
assertQuery("abc AND xyz OR olingo").resultsIn(WORD, AND, WORD, OR, WORD);
|
||||
assertQuery("abc AND ANDsomething").addExpected(WORD, AND, WORD);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void parseCombinations() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
assertQuery("word O NO").resultsIn(word("word"), word("O"), word("NO"));
|
||||
assertQuery("O AN NO").resultsIn(word("O"), word("AN"), word("NO"));
|
||||
assertQuery("NO AN O").resultsIn(word("NO"), word("AN"), word("O"));
|
||||
assertQuery("N A O").resultsIn(word("N"), word("A"), word("O"));
|
||||
assertQuery("abc AND NOT xyz OR olingo").resultsIn(WORD, AND, NOT, WORD, OR, WORD);
|
||||
|
||||
result = tokenizer.tokenize("abc AND NOT xyz OR olingo");
|
||||
Assert.assertNotNull(result);
|
||||
|
||||
Iterator<SearchQueryToken> it = result.iterator();
|
||||
Assert.assertEquals(WORD, it.next().getToken());
|
||||
Assert.assertEquals(AND, it.next().getToken());
|
||||
Assert.assertEquals(NOT, it.next().getToken());
|
||||
Assert.assertEquals(WORD, it.next().getToken());
|
||||
Assert.assertEquals(OR, it.next().getToken());
|
||||
Assert.assertEquals(WORD, it.next().getToken());
|
||||
|
||||
TokenizerValidator.init("foo AND bar OR foo AND baz OR that AND bar OR that AND baz")
|
||||
assertQuery("foo AND bar OR foo AND baz OR that AND bar OR that AND baz")
|
||||
.addExpected(WORD, "foo").addExpected(AND)
|
||||
.addExpected(WORD, "bar").addExpected(OR)
|
||||
.addExpected(WORD, "foo").addExpected(AND)
|
||||
|
|
@ -294,7 +200,7 @@ public class SearchTokenizerTest {
|
|||
.validate();
|
||||
|
||||
|
||||
TokenizerValidator.init("(foo OR that) AND (bar OR baz)")
|
||||
assertQuery("(foo OR that) AND (bar OR baz)")
|
||||
.addExpected(OPEN)
|
||||
.addExpected(WORD, "foo").addExpected(OR).addExpected(WORD, "that")
|
||||
.addExpected(CLOSE).addExpected(AND).addExpected(OPEN)
|
||||
|
|
@ -306,47 +212,21 @@ public class SearchTokenizerTest {
|
|||
|
||||
@Test
|
||||
public void parseSpecial() throws Exception {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result;
|
||||
Iterator<SearchQueryToken> it;
|
||||
|
||||
result = tokenizer.tokenize("NOT abc AND nothing");
|
||||
|
||||
it = result.iterator();
|
||||
Assert.assertEquals(NOT, it.next().getToken());
|
||||
Assert.assertEquals(WORD, it.next().getToken());
|
||||
Assert.assertEquals(AND, it.next().getToken());
|
||||
Assert.assertEquals(WORD, it.next().getToken());
|
||||
|
||||
result = tokenizer.tokenize("abc AND andsomething");
|
||||
|
||||
it = result.iterator();
|
||||
Assert.assertEquals(WORD, it.next().getToken());
|
||||
Assert.assertEquals(AND, it.next().getToken());
|
||||
Assert.assertEquals(WORD, it.next().getToken());
|
||||
|
||||
TokenizerValidator.init("abc AND ANDsomething")
|
||||
.addExpected(WORD, AND, WORD).validate();
|
||||
|
||||
TokenizerValidator.init("abc ANDsomething")
|
||||
.addExpected(WORD, WORD).validate();
|
||||
|
||||
TokenizerValidator.init("abc ORsomething")
|
||||
.addExpected(WORD, WORD).validate();
|
||||
|
||||
TokenizerValidator.init("abc OR orsomething")
|
||||
.addExpected(WORD, OR, WORD).validate();
|
||||
|
||||
TokenizerValidator.init("abc OR ORsomething")
|
||||
.addExpected(WORD, OR, WORD).validate();
|
||||
assertQuery("NOT abc AND nothing").resultsIn(NOT, WORD, AND, WORD);
|
||||
assertQuery("abc AND andsomething").resultsIn(WORD, AND, WORD);
|
||||
assertQuery("abc AND ANDsomething").resultsIn(WORD, AND, WORD);
|
||||
assertQuery("abc ANDsomething").resultsIn(WORD, WORD);
|
||||
assertQuery("abc ORsomething").resultsIn(WORD, WORD);
|
||||
assertQuery("abc OR orsomething").resultsIn(WORD, OR, WORD);
|
||||
assertQuery("abc OR ORsomething").resultsIn(WORD, OR, WORD);
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void unicodeInWords() throws Exception {
|
||||
// Ll, Lm, Lo, Lt, Lu, Nl
|
||||
TokenizerValidator.init("abc OR Ll\u01E3Lm\u02B5Lo\u1BE4Lt\u01F2Lu\u03D3Nl\u216F")
|
||||
.addExpected(WORD, OR, WORD).validate();
|
||||
assertQuery("abc OR Ll\u01E3Lm\u02B5Lo\u1BE4Lt\u01F2Lu\u03D3Nl\u216F")
|
||||
.resultsIn(WORD, OR, WORD);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -369,8 +249,7 @@ public class SearchTokenizerTest {
|
|||
*/
|
||||
@Test
|
||||
public void characterInPhrase() throws Exception {
|
||||
TokenizerValidator.init("\"123\" OR \"ALPHA-._~\"")
|
||||
.addExpected(PHRASE, OR, PHRASE).validate();
|
||||
assertQuery("\"123\" OR \"ALPHA-._~\"").resultsIn(PHRASE, OR, PHRASE);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -395,7 +274,7 @@ public class SearchTokenizerTest {
|
|||
validate("abc def ghi");
|
||||
|
||||
// mixed not
|
||||
TokenizerValidator.init(" abc def AND ghi").validate(WORD, WORD, AND, WORD);
|
||||
assertQuery(" abc def AND ghi").resultsIn(WORD, WORD, AND, WORD);
|
||||
validate("NOT abc NOT def OR NOT ghi", NOT, WORD, NOT, WORD, OR, NOT, WORD);
|
||||
validate(" abc def NOT ghi", WORD, WORD, NOT, WORD);
|
||||
|
||||
|
|
@ -411,48 +290,60 @@ public class SearchTokenizerTest {
|
|||
@Test
|
||||
public void tokenizeInvalid() throws SearchTokenizerException {
|
||||
//
|
||||
TokenizerValidator.init("( abc AND) OR something").validate(SearchTokenizerException.class);
|
||||
assertQuery("( abc AND) OR something").resultsIn(SearchTokenizerException.class);
|
||||
|
||||
TokenizerValidator.init("\"phrase\"word").validate(SearchTokenizerException.class);
|
||||
TokenizerValidator.init("\"p\"w").validate(SearchTokenizerException.class);
|
||||
TokenizerValidator.init("\"\"").validate(SearchTokenizerException.class);
|
||||
assertQuery("\"phrase\"word").resultsIn(SearchTokenizerException.class);
|
||||
assertQuery("\"p\"w").resultsIn(SearchTokenizerException.class);
|
||||
assertQuery("\"\"").resultsIn(SearchTokenizerException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void tokenizeInvalidQueryForParser() throws SearchTokenizerException {
|
||||
TokenizerValidator.init("AND").validate(AND);
|
||||
TokenizerValidator.init("OR").validate(OR);
|
||||
TokenizerValidator.init("NOT").validate(NOT);
|
||||
TokenizerValidator.init("NOT AND").validate(NOT, AND);
|
||||
TokenizerValidator.init("NOT OR").validate(NOT, OR);
|
||||
TokenizerValidator.init("NOT NOT").validate(NOT, NOT);
|
||||
TokenizerValidator.init("abc AND OR something").validate(WORD, AND, OR, WORD);
|
||||
TokenizerValidator.init("abc AND \"something\" )").validate(WORD, AND, PHRASE, CLOSE);
|
||||
assertQuery("AND").resultsIn(AND);
|
||||
assertQuery("OR").resultsIn(OR);
|
||||
assertQuery("NOT").resultsIn(NOT);
|
||||
assertQuery("NOT AND").resultsIn(NOT, AND);
|
||||
assertQuery("NOT OR").resultsIn(NOT, OR);
|
||||
assertQuery("NOT NOT").resultsIn(NOT, NOT);
|
||||
assertQuery("abc AND OR something").resultsIn(WORD, AND, OR, WORD);
|
||||
assertQuery("abc AND \"something\" )").resultsIn(WORD, AND, PHRASE, CLOSE);
|
||||
}
|
||||
|
||||
public void validate(String query) throws SearchTokenizerException {
|
||||
new TokenizerValidator(query).validate();
|
||||
new Validator(query);
|
||||
}
|
||||
|
||||
public Validator assertQuery(String query) throws SearchTokenizerException {
|
||||
return new Validator(query);
|
||||
}
|
||||
|
||||
public void validate(String query, SearchQueryToken.Token ... tokens) throws SearchTokenizerException {
|
||||
TokenizerValidator sv = new TokenizerValidator(query);
|
||||
Validator sv = new Validator(query);
|
||||
for (SearchQueryToken.Token token : tokens) {
|
||||
sv.addExpected(token);
|
||||
}
|
||||
sv.validate();
|
||||
}
|
||||
|
||||
private static class TokenizerValidator {
|
||||
private static class Validator {
|
||||
private List<Tuple> validations = new ArrayList<Tuple>();
|
||||
private boolean log;
|
||||
private final String searchQuery;
|
||||
|
||||
public void validate(SearchQueryToken.Token... tokens) throws SearchTokenizerException {
|
||||
public void resultsIn(SearchQueryToken.Token... tokens) throws SearchTokenizerException {
|
||||
addExpected(tokens);
|
||||
validate();
|
||||
}
|
||||
|
||||
private class Tuple {
|
||||
public void resultsIn(Tuple... tuple) throws SearchTokenizerException {
|
||||
for (Tuple t : tuple) {
|
||||
addExpected(t.token, t.literal);
|
||||
}
|
||||
validate();
|
||||
}
|
||||
public static Tuple tuple(SearchQueryToken.Token token, String literal) {
|
||||
return new Tuple(token, literal);
|
||||
}
|
||||
private static class Tuple {
|
||||
final SearchQueryToken.Token token;
|
||||
final String literal;
|
||||
public Tuple(SearchQueryToken.Token token, String literal) {
|
||||
|
|
@ -464,30 +355,30 @@ public class SearchTokenizerTest {
|
|||
}
|
||||
}
|
||||
|
||||
private TokenizerValidator(String searchQuery) {
|
||||
private Validator(String searchQuery) {
|
||||
this.searchQuery = searchQuery;
|
||||
}
|
||||
|
||||
private static TokenizerValidator init(String searchQuery) {
|
||||
return new TokenizerValidator(searchQuery);
|
||||
private static Validator init(String searchQuery) {
|
||||
return new Validator(searchQuery);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
private TokenizerValidator enableLogging() {
|
||||
private Validator enableLogging() {
|
||||
log = true;
|
||||
return this;
|
||||
}
|
||||
private TokenizerValidator addExpected(SearchQueryToken.Token token, String literal) {
|
||||
private Validator addExpected(SearchQueryToken.Token token, String literal) {
|
||||
validations.add(new Tuple(token, literal));
|
||||
return this;
|
||||
}
|
||||
private TokenizerValidator addExpected(SearchQueryToken.Token ... token) {
|
||||
private Validator addExpected(SearchQueryToken.Token ... token) {
|
||||
for (SearchQueryToken.Token t : token) {
|
||||
validations.add(new Tuple(t));
|
||||
}
|
||||
return this;
|
||||
}
|
||||
private void validate(Class<? extends Exception> exception) throws SearchTokenizerException {
|
||||
private void resultsIn(Class<? extends Exception> exception) throws SearchTokenizerException {
|
||||
try {
|
||||
new SearchTokenizer().tokenize(searchQuery);
|
||||
} catch (Exception e) {
|
||||
|
|
@ -497,6 +388,17 @@ public class SearchTokenizerTest {
|
|||
Assert.fail("Expected exception " + exception.getClass().getSimpleName() + " was not thrown.");
|
||||
}
|
||||
|
||||
private void resultsIn(SearchTokenizerException.MessageKey key)
|
||||
throws SearchTokenizerException {
|
||||
try {
|
||||
init(searchQuery).validate();
|
||||
} catch (SearchTokenizerException e) {
|
||||
Assert.assertEquals("SearchTokenizerException with unexpected message was thrown.", key, e.getMessageKey());
|
||||
return;
|
||||
}
|
||||
Assert.fail("No SearchTokenizerException was not thrown.");
|
||||
}
|
||||
|
||||
private void validate() throws SearchTokenizerException {
|
||||
SearchTokenizer tokenizer = new SearchTokenizer();
|
||||
List<SearchQueryToken> result = tokenizer.tokenize(searchQuery);
|
||||
|
|
|
|||
Loading…
Reference in New Issue