[OLINGO-568] More tests and fixes

lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java

@@ -129,6 +129,9 @@ public class SearchParser {
     SearchExpression se = left;
     if (isTerm()) {
       se = processTerm();
+      if(isTerm()) {
+        se = processAnd(se);
+      }
       se = new SearchBinaryImpl(left, SearchBinaryOperatorKind.AND, se);
       return processSearchExpression(se);
     } else {
@@ -154,6 +157,10 @@ public class SearchParser {
     if (isToken(Token.WORD) || isToken(Token.PHRASE)) {
       return new SearchUnaryImpl(processWordOrPhrase());
     }
+    if(isEof()) {
+      throw new SearchParserException("NOT must be followed by a term.",
+          SearchParserException.MessageKeys.INVALID_NOT_OPERAND, "EOF");
+    }
     throw new SearchParserException("NOT must be followed by a term not a " + token.getToken(),
         SearchParserException.MessageKeys.INVALID_NOT_OPERAND, token.getToken().toString());
   }

lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java

@@ -76,7 +76,7 @@ public class SearchTokenizer {
     }
 
     public State invalid() throws SearchTokenizerException {
-      throw new SearchTokenizerException("Token " + this.getToken() + " is in invalid state ",
+      throw new SearchTokenizerException("Token " + this.getToken() + " is in invalid state.",
           SearchTokenizerException.MessageKeys.INVALID_TOKEN_STATE);
     }
 
@@ -428,7 +428,7 @@ public class SearchTokenizer {
       if(closed) {
         return finish();
       }
-      return super.close();
+      return invalid();
     }
   }
 
@@ -475,15 +475,18 @@ public class SearchTokenizer {
       } else if (literal.length() == 3 && isWhitespace(c)) {
         finish();
         return new BeforePhraseOrWordRwsState();
+      } else if(isWhitespace(c)) {
+        changeToken(Token.WORD).finish();
+        return new RwsState();
       }
-      return forbidden(c);
+      return new SearchWordState(this).nextChar(c);
     }
     @Override
     public State close() throws SearchTokenizerException {
       if(Token.NOT.name().equals(literal.toString())) {
         return finish();
       }
-      return super.close();
+      return changeToken(Token.WORD).finish();
     }
   }
 
@@ -504,9 +507,18 @@ public class SearchTokenizer {
       } else if (literal.length() == 3 && isWhitespace(c)) {
         finish();
         return new BeforeSearchExpressionRwsState();
-      } else {
+      } else if(isWhitespace(c)) {
-        return new SearchWordState(this);
+        changeToken(Token.WORD).finish();
+        return new RwsState();
       }
+      return new SearchWordState(this).nextChar(c);
+    }
+    @Override
+    public State close() throws SearchTokenizerException {
+      if(Token.AND.name().equals(literal.toString())) {
+        return finish();
+      }
+      return changeToken(Token.WORD).finish();
     }
   }
 
@@ -517,7 +529,6 @@ public class SearchTokenizer {
         forbidden(c);
       }
     }
-
     @Override
     public State nextChar(char c) throws SearchTokenizerException {
       if (literal.length() == 1 && (c == CHAR_R)) {
@@ -525,9 +536,18 @@ public class SearchTokenizer {
       } else if (literal.length() == 2 && isWhitespace(c)) {
         finish();
         return new BeforeSearchExpressionRwsState();
-      } else {
+      } else if(isWhitespace(c)) {
-        return new SearchWordState(this);
+        changeToken(Token.WORD).finish();
+        return new RwsState();
       }
+      return new SearchWordState(this).nextChar(c);
+    }
+    @Override
+    public State close() throws SearchTokenizerException {
+      if(Token.OR.name().equals(literal.toString())) {
+        return finish();
+      }
+      return changeToken(Token.WORD).finish();
     }
   }
 

lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java

@@ -18,15 +18,9 @@
  */
 package org.apache.olingo.server.core.uri.parser.search;
 
-import static org.apache.olingo.server.api.uri.queryoption.search.SearchBinaryOperatorKind.AND;
-import static org.apache.olingo.server.api.uri.queryoption.search.SearchBinaryOperatorKind.OR;
-
-import java.lang.reflect.Field;
-
 import org.apache.olingo.server.api.ODataLibraryException;
 import org.apache.olingo.server.api.uri.queryoption.SearchOption;
 import org.apache.olingo.server.api.uri.queryoption.search.SearchExpression;
-import org.apache.olingo.server.api.uri.queryoption.search.SearchUnary;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -34,149 +28,86 @@ public class SearchParserAndTokenizerTest {
 
   @Test
   public void basicParsing() throws Exception {
-    SearchExpressionValidator.init("\"99\"")
+    assertQuery("\"99\"").resultsIn("'99'");
-            .validate(with("99"));
+    assertQuery("a").resultsIn("'a'");
-    SearchExpressionValidator.init("a")
+    assertQuery("a AND b").resultsIn("{'a' AND 'b'}");
-        .validate(with("a"));
+    assertQuery("a AND b AND c").resultsIn("{{'a' AND 'b'} AND 'c'}");
-    SearchExpressionValidator.init("a AND b")
+    assertQuery("a OR b").resultsIn("{'a' OR 'b'}");
-        .validate(with("a", and("b")));
+    assertQuery("a OR b OR c").resultsIn("{'a' OR {'b' OR 'c'}}");
-    SearchExpressionValidator.init("a AND b AND c")
-        .validate("{{'a' AND 'b'} AND 'c'}");
-    SearchExpressionValidator.init("a OR b")
-        .validate(with("a", or("b")));
-    SearchExpressionValidator.init("a OR b OR c")
-        .validate(with("a", or("b", or("c"))));
   }
 
   @Test
   public void mixedParsing() throws Exception {
-    SearchExpressionValidator.init("a AND b OR c")
+    assertQuery("a AND b OR c").resultsIn("{{'a' AND 'b'} OR 'c'}");
-        .validate("{{'a' AND 'b'} OR 'c'}");
+    assertQuery("a OR b AND c").resultsIn("{'a' OR {'b' AND 'c'}}");
-    SearchExpressionValidator.init("a OR b AND c")
-        .validate("{'a' OR {'b' AND 'c'}}");
   }
 
   @Test
   public void notParsing() throws Exception {
-    SearchExpressionValidator.init("NOT a AND b OR c")
+    assertQuery("NOT a AND b OR c").resultsIn("{{{NOT 'a'} AND 'b'} OR 'c'}");
-        .validate("{{{NOT 'a'} AND 'b'} OR 'c'}");
+    assertQuery("a OR b AND NOT c").resultsIn("{'a' OR {'b' AND {NOT 'c'}}}");
-    SearchExpressionValidator.init("a OR b AND NOT c")
-        .validate("{'a' OR {'b' AND {NOT 'c'}}}");
   }
 
   @Test
   public void parenthesesParsing() throws Exception {
-    SearchExpressionValidator.init("a AND (b OR c)")
+    assertQuery("a AND (b OR c)").resultsIn("{'a' AND {'b' OR 'c'}}");
-        .validate("{'a' AND {'b' OR 'c'}}");
+    assertQuery("(a OR b) AND NOT c").resultsIn("{{'a' OR 'b'} AND {NOT 'c'}}");
-    SearchExpressionValidator.init("(a OR b) AND NOT c")
+  }
-        .validate("{{'a' OR 'b'} AND {NOT 'c'}}");
+
+  @Test
+  public void parseImplicitAnd() throws Exception {
+    assertQuery("a b").resultsIn("{'a' AND 'b'}");
+    assertQuery("a b c").resultsIn("{'a' AND {'b' AND 'c'}}");
+    assertQuery("a and b").resultsIn("{'a' AND {'and' AND 'b'}}");
+    assertQuery("a b OR c").resultsIn("{{'a' AND 'b'} OR 'c'}");
+    assertQuery("a \"bc123\" OR c").resultsIn("{{'a' AND 'bc123'} OR 'c'}");
+    assertQuery("(a OR x) bc c").resultsIn("{{'a' OR 'x'} AND {'bc' AND 'c'}}");
+    assertQuery("one ((a OR x) bc c)").resultsIn("{'one' AND {{'a' OR 'x'} AND {'bc' AND 'c'}}}");
   }
 
   @Test
   public void invalidSearchQuery() throws Exception {
-    SearchExpressionValidator.init("99").validate(SearchParserException.class,
+    assertQuery("99").resultsIn(SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
-            SearchParserException.MessageKeys.TOKENIZER_EXCEPTION);
+    assertQuery("NOT").resultsIn(SearchParserException.MessageKeys.INVALID_NOT_OPERAND);
+    assertQuery("AND").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION);
+    assertQuery("OR").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION);
   }
 
-  private static SearchExpression with(String term) {
+  private static Validator assertQuery(String searchQuery) {
-    return new SearchTermImpl(term);
+    return Validator.init(searchQuery);
   }
 
-  private static SearchExpression with(String left, SearchExpression right) {
+  private static class Validator {
-    setLeftField(left, right);
-    return right;
-  }
-
-  private static SearchUnary with(SearchUnary unary) {
-    return unary;
-  }
-
-  private static SearchExpression or(String left, SearchExpression right) {
-    SearchExpression or = or(right);
-    setLeftField(left, right);
-    return or;
-  }
-
-  private static SearchExpression and(String left, SearchExpression right) {
-    SearchExpression and = and(right);
-    setLeftField(left, right);
-    return and;
-  }
-
-  private static SearchExpression or(SearchExpression right) {
-    return new SearchBinaryImpl(null, OR, right);
-  }
-
-  private static SearchExpression and(SearchExpression right) {
-    return new SearchBinaryImpl(null, AND, right);
-  }
-
-  private static SearchExpression and(String right) {
-    return and(new SearchTermImpl(right));
-  }
-
-  private static SearchExpression or(String right) {
-    return or(new SearchTermImpl(right));
-  }
-
-  private static SearchUnary not(String term) {
-    return new SearchUnaryImpl(new SearchTermImpl(term));
-  }
-
-  private static void setLeftField(String left, SearchExpression se) {
-    try {
-      Field field = null;
-      if (se instanceof SearchUnaryImpl) {
-        field = SearchBinaryImpl.class.getDeclaredField("operand");
-      } else if (se instanceof SearchBinaryImpl) {
-        field = SearchBinaryImpl.class.getDeclaredField("left");
-      } else {
-        Assert.fail("Unexpected exception: " + se.getClass());
-      }
-      field.setAccessible(true);
-      field.set(se, new SearchTermImpl(left));
-    } catch (Exception e) {
-      Assert.fail("Unexpected exception: " + e.getClass());
-    }
-  }
-
-  private static class SearchExpressionValidator {
     private boolean log;
     private final String searchQuery;
 
-    private SearchExpressionValidator(String searchQuery) {
+    private Validator(String searchQuery) {
       this.searchQuery = searchQuery;
     }
 
-    private static SearchExpressionValidator init(String searchQuery) {
+    private static Validator init(String searchQuery) {
-      return new SearchExpressionValidator(searchQuery);
+      return new Validator(searchQuery);
     }
 
     @SuppressWarnings("unused")
-    private SearchExpressionValidator enableLogging() {
+    private Validator withLogging() {
       log = true;
       return this;
     }
 
-    private void validate(Class<? extends ODataLibraryException> exception, ODataLibraryException.MessageKey key)
+    private void resultsIn(ODataLibraryException.MessageKey key)
             throws SearchTokenizerException {
       try {
-        validate(searchQuery);
+        resultsIn(searchQuery);
       } catch (ODataLibraryException e) {
-        Assert.assertEquals(exception, e.getClass());
+        Assert.assertEquals(SearchParserException.class, e.getClass());
         Assert.assertEquals(key, e.getMessageKey());
         return;
       }
-      Assert.fail("Expected exception " + exception.getClass().getSimpleName() + " was not thrown.");
+      Assert.fail("SearchParserException with message key " + key.getKey() + " was not thrown.");
     }
 
-    private void validate(SearchExpression expectedSearchExpression) throws SearchTokenizerException,
+    private void resultsIn(String expectedSearchExpression) throws SearchTokenizerException, SearchParserException {
-        SearchParserException {
-      final SearchExpression searchExpression = getSearchExpression();
-      Assert.assertEquals(expectedSearchExpression.toString(), searchExpression.toString());
-    }
-
-    private void validate(String expectedSearchExpression) throws SearchTokenizerException, SearchParserException {
       final SearchExpression searchExpression = getSearchExpression();
       Assert.assertEquals(expectedSearchExpression, searchExpression.toString());
     }

lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java

@@ -46,7 +46,6 @@ public class SearchParserTest extends SearchParser {
     se = run(Token.PHRASE);
     assertEquals("'phrase1'", se.toString());
     assertTrue(se.isSearchTerm());
-    // TODO: Check if quotation marks should be part of the string we deliver
     assertEquals("phrase1", se.asSearchTerm().getSearchTerm());
   }
 

lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java

@@ -38,52 +38,28 @@ public class SearchTokenizerTest {
 
   @Test
   public void parseBasics() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("abd").resultsIn(WORD);
-    List<SearchQueryToken> result;
+    assertQuery("NOT abc").resultsIn(NOT, WORD);
-    
+    assertQuery("(abc)").resultsIn(OPEN, WORD, CLOSE);
-    //
+    assertQuery("((abc))").resultsIn(OPEN, OPEN, WORD, CLOSE, CLOSE);
-    result = tokenizer.tokenize("abc");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-
-    result = tokenizer.tokenize("NOT abc");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(NOT, result.get(0).getToken());
-    Assert.assertEquals(WORD, result.get(1).getToken());
-
-    result = tokenizer.tokenize("(abc)");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(OPEN, result.get(0).getToken());
-    Assert.assertEquals(WORD, result.get(1).getToken());
-    Assert.assertEquals(CLOSE, result.get(2).getToken());
-
-    result = tokenizer.tokenize("((abc))");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(OPEN, result.get(0).getToken());
-    Assert.assertEquals(WORD, result.get(2).getToken());
-    Assert.assertEquals(CLOSE, result.get(4).getToken());
   }
 
   @Test
   public void parseWords() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("somesimpleword").resultsIn(WORD);
-    List<SearchQueryToken> result;
+    assertQuery("anotherWord\u1234").resultsIn(WORD);
+    // special
+    assertQuery("NO").resultsIn(word("NO"));
+    assertQuery("N").resultsIn(word("N"));
+    assertQuery("A").resultsIn(word("A"));
+    assertQuery("AN").resultsIn(word("AN"));
+    assertQuery("O").resultsIn(word("O"));
+    // invalid
+    assertQuery("notAw0rd").resultsIn(SearchTokenizerException.MessageKeys.FORBIDDEN_CHARACTER);
+  }
 
-    //
+  private Validator.Tuple word(String literal) {
-    result = tokenizer.tokenize("abc");
+    return Validator.tuple(WORD, literal);
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-
-    //
-    result = tokenizer.tokenize("anotherWord\u1234");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
   }
 
   @Test
@@ -91,7 +67,7 @@ public class SearchTokenizerTest {
     SearchTokenizer tokenizer = new SearchTokenizer();
     List<SearchQueryToken> result;
 
-    TokenizerValidator.init("abc AND \"x-y_z\" AND olingo").validate();
+    assertQuery("abc AND \"x-y_z\" AND olingo");
 
     //
     result = tokenizer.tokenize("\"abc\"");
@@ -113,7 +89,7 @@ public class SearchTokenizerTest {
     Assert.assertEquals(PHRASE, result.get(0).getToken());
     Assert.assertEquals("\"99_88.\"", result.get(0).getLiteral());
 
-    TokenizerValidator.init("abc or \"xyz\"").validate(WORD, WORD, PHRASE);
+    assertQuery("abc or \"xyz\"").resultsIn(WORD, WORD, PHRASE);
   }
 
   /**
@@ -124,165 +100,95 @@ public class SearchTokenizerTest {
   @Ignore("Test must be moved to SearchParserTest and SearchParserAndTokenizerTest")
   public void parsePhraseAbnfTestcases() throws Exception {
     //    <TestCase Name="5.1.7 Search - simple phrase" Rule="queryOptions">
-    TokenizerValidator.init("\"blue%20green\"").validate();
+    assertQuery("\"blue%20green\"");
     //    <TestCase Name="5.1.7 Search - simple phrase" Rule="queryOptions">
-    TokenizerValidator.init("\"blue%20green%22").validate();
+    assertQuery("\"blue%20green%22");
     //    <TestCase Name="5.1.7 Search - phrase with escaped double-quote" Rule="queryOptions">
     //    <Input>$search="blue\"green"</Input>
-    TokenizerValidator.init("\"blue\\\"green\"").validate();
+    assertQuery("\"blue\\\"green\"");
 
     //    <TestCase Name="5.1.7 Search - phrase with escaped backslash" Rule="queryOptions">
     //    <Input>$search="blue\\green"</Input>
-    TokenizerValidator.init("\"blue\\\\green\"").validate();
+    assertQuery("\"blue\\\\green\"");
 
     //    <TestCase Name="5.1.7 Search - phrase with unescaped double-quote" Rule="queryOptions" FailAt="14">
-    TokenizerValidator.init("\"blue\"green\"").validate();
+    assertQuery("\"blue\"green\"");
 
     //    <TestCase Name="5.1.7 Search - phrase with unescaped double-quote" Rule="queryOptions" FailAt="16">
-    TokenizerValidator.init("\"blue%22green\"").validate();
+    assertQuery("\"blue%22green\"");
 
 //    <TestCase Name="5.1.7 Search - implicit AND" Rule="queryOptions">
 //    <Input>$search=blue green</Input>
-//    SearchValidator.init("\"blue%20green\"").validate();
+//    SearchassertQuery("\"blue%20green\"").resultsIn();
     //    <TestCase Name="5.1.7 Search - implicit AND, encoced" Rule="queryOptions">
-//    SearchValidator.init("blue%20green").validate();
+//    SearchassertQuery("blue%20green").resultsIn();
   }
 
 
     @Test
   public void parseNot() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("NOT").resultsIn(NOT);
-    List<SearchQueryToken> result;
+    assertQuery(" NOT ").resultsIn(NOT);
-
+    assertQuery("NOT abc").resultsIn(NOT, WORD);
-    result = tokenizer.tokenize("NOT abc");
+    assertQuery("not abc").resultsIn(WORD, WORD);
-    Assert.assertNotNull(result);
+    assertQuery("NOT    abc").resultsIn(NOT, WORD);
-    
+    assertQuery("NOT    \"abc\"").resultsIn(NOT, PHRASE);
-    Assert.assertEquals(NOT, result.get(0).getToken());
+    assertQuery("NObody").resultsIn(WORD);
-    Assert.assertEquals(WORD, result.get(1).getToken());
+    assertQuery("Nobody").resultsIn(WORD);
-
+    assertQuery("NOT (sdf)").resultsIn(SearchTokenizerException.MessageKeys.FORBIDDEN_CHARACTER);
-    TokenizerValidator.init("not abc").addExpected(WORD, WORD).validate();
-    TokenizerValidator.init("NOT    abc").addExpected(NOT, WORD).validate();
-    TokenizerValidator.init("NOT    \"abc\"").addExpected(NOT, PHRASE).validate();
-    TokenizerValidator.init("NOT (sdf)").validate(SearchTokenizerException.class);
   }
 
   @Test
   public void parseOr() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("OR").resultsIn(OR);
-    List<SearchQueryToken> result;
+    assertQuery(" OR ").resultsIn(OR);
-
+    assertQuery("OR xyz").resultsIn(OR, WORD);
-    result = tokenizer.tokenize("abc OR xyz");
+    assertQuery("abc OR xyz").resultsIn(WORD, OR, WORD);
-    Assert.assertNotNull(result);
+    assertQuery("abc OR xyz OR olingo").resultsIn(WORD, OR, WORD, OR, WORD);
-    
+    assertQuery("abc or xyz").addExpected(WORD, WORD, WORD);
-    Assert.assertEquals(WORD, result.get(0).getToken());
-    Assert.assertEquals(OR, result.get(1).getToken());
-    Assert.assertEquals(WORD, result.get(2).getToken());
-
-    result = tokenizer.tokenize("abc OR xyz OR olingo");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-    Assert.assertEquals(OR, result.get(1).getToken());
-    Assert.assertEquals(WORD, result.get(2).getToken());
-    Assert.assertEquals(OR, result.get(3).getToken());
-    Assert.assertEquals(WORD, result.get(4).getToken());
-
-    TokenizerValidator.init("abc or xyz").addExpected(WORD, WORD, WORD).validate();
   }
 
   @Test
   public void parseImplicitAnd() throws SearchTokenizerException {
-    TokenizerValidator.init("a b").addExpected(WORD, WORD).validate();
+    assertQuery("a b").resultsIn(WORD, WORD);
-    TokenizerValidator.init("a b OR c").addExpected(WORD, WORD, OR, WORD).validate();
+    assertQuery("a b OR c").resultsIn(WORD, WORD, OR, WORD);
-    TokenizerValidator.init("a bc OR c").addExpected(WORD, WORD, OR, WORD).validate();
+    assertQuery("a bc OR c").resultsIn(WORD, WORD, OR, WORD);
-    TokenizerValidator.init("a bc c").addExpected(WORD, WORD, WORD).validate();
+    assertQuery("a bc c").resultsIn(WORD, WORD, WORD);
-    TokenizerValidator.init("(a OR x) bc c").addExpected(OPEN, WORD, OR, WORD, CLOSE, WORD, WORD).validate();
+    assertQuery("(a OR x) bc c").resultsIn(OPEN, WORD, OR, WORD, CLOSE, WORD, WORD);
   }
 
   @Test
   public void parseAnd() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("AND").resultsIn(AND);
-    List<SearchQueryToken> result;
+    assertQuery(" AND ").resultsIn(AND);
-
-    result = tokenizer.tokenize("abc AND xyz");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-    Assert.assertEquals(AND, result.get(1).getToken());
-    Assert.assertEquals(WORD, result.get(2).getToken());
 
+    assertQuery("abc AND xyz").resultsIn(WORD, AND, WORD);
     // no lower case allowed for AND
-    result = tokenizer.tokenize("abc and xyz");
+    assertQuery("abc and xyz").resultsIn(WORD, WORD, WORD);
-    Assert.assertNotNull(result);
+    // implicit AND is handled by parser (and not tokenizer)
-    Assert.assertEquals(3, result.size());
+    assertQuery("abc xyz").resultsIn(WORD, WORD);
-    
+    assertQuery("abc AND xyz AND olingo").resultsIn(WORD, AND, WORD, AND, WORD);
-    Assert.assertEquals(WORD, result.get(0).getToken());
+    assertQuery("abc AND \"x-y_z\"  AND olingo")
-    Assert.assertEquals(WORD, result.get(1).getToken());
+        .resultsIn(WORD, AND, PHRASE, AND, WORD);
-    Assert.assertEquals(WORD, result.get(2).getToken());
-
-    // implicit AND
-    result = tokenizer.tokenize("abc xyz");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-    Assert.assertEquals(WORD, result.get(1).getToken());
-
-    result = tokenizer.tokenize("abc AND xyz AND olingo");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-    Assert.assertEquals(AND, result.get(1).getToken());
-    Assert.assertEquals(WORD, result.get(2).getToken());
-    Assert.assertEquals(AND, result.get(3).getToken());
-    Assert.assertEquals(WORD, result.get(4).getToken());
-
-    result = tokenizer.tokenize("abc AND \"x-y_z\" AND olingo");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-    Assert.assertEquals(AND, result.get(1).getToken());
-    Assert.assertEquals(PHRASE, result.get(2).getToken());
-    Assert.assertEquals("\"x-y_z\"", result.get(2).getLiteral());
-    Assert.assertEquals(AND, result.get(3).getToken());
-    Assert.assertEquals(WORD, result.get(4).getToken());
   }
 
   @Test
   public void parseAndOr() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("OR AND ").resultsIn(OR, AND);
-    List<SearchQueryToken> result;
+    assertQuery("abc AND xyz OR olingo").resultsIn(WORD, AND, WORD, OR, WORD);
-
+    assertQuery("abc AND ANDsomething").addExpected(WORD, AND, WORD);
-    result = tokenizer.tokenize("abc AND xyz OR olingo");
-    Assert.assertNotNull(result);
-    
-    Assert.assertEquals(WORD, result.get(0).getToken());
-    Assert.assertEquals(AND, result.get(1).getToken());
-    Assert.assertEquals(WORD, result.get(2).getToken());
-    Assert.assertEquals(OR, result.get(3).getToken());
-    Assert.assertEquals(WORD, result.get(4).getToken());
-
-    TokenizerValidator.init("abc AND ANDsomething")
-        .addExpected(WORD, AND, WORD).validate();
   }
 
 
   @Test
   public void parseCombinations() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("word O NO").resultsIn(word("word"), word("O"), word("NO"));
-    List<SearchQueryToken> result;
+    assertQuery("O AN NO").resultsIn(word("O"), word("AN"), word("NO"));
+    assertQuery("NO AN O").resultsIn(word("NO"), word("AN"), word("O"));
+    assertQuery("N A O").resultsIn(word("N"), word("A"), word("O"));
+    assertQuery("abc AND NOT xyz OR olingo").resultsIn(WORD, AND, NOT, WORD, OR, WORD);
 
-    result = tokenizer.tokenize("abc AND NOT xyz OR olingo");
+    assertQuery("foo AND bar OR foo AND baz OR that AND bar OR that AND baz")
-    Assert.assertNotNull(result);
-    
-    Iterator<SearchQueryToken> it = result.iterator();
-    Assert.assertEquals(WORD, it.next().getToken());
-    Assert.assertEquals(AND, it.next().getToken());
-    Assert.assertEquals(NOT, it.next().getToken());
-    Assert.assertEquals(WORD, it.next().getToken());
-    Assert.assertEquals(OR, it.next().getToken());
-    Assert.assertEquals(WORD, it.next().getToken());
-
-    TokenizerValidator.init("foo AND bar OR foo AND baz OR that AND bar OR that AND baz")
         .addExpected(WORD, "foo").addExpected(AND)
         .addExpected(WORD, "bar").addExpected(OR)
         .addExpected(WORD, "foo").addExpected(AND)
@@ -294,7 +200,7 @@ public class SearchTokenizerTest {
         .validate();
 
 
-    TokenizerValidator.init("(foo OR that) AND (bar OR baz)")
+    assertQuery("(foo OR that) AND (bar OR baz)")
         .addExpected(OPEN)
         .addExpected(WORD, "foo").addExpected(OR).addExpected(WORD, "that")
         .addExpected(CLOSE).addExpected(AND).addExpected(OPEN)
@@ -306,47 +212,21 @@ public class SearchTokenizerTest {
 
   @Test
   public void parseSpecial() throws Exception {
-    SearchTokenizer tokenizer = new SearchTokenizer();
+    assertQuery("NOT abc AND nothing").resultsIn(NOT, WORD, AND, WORD);
-    List<SearchQueryToken> result;
+    assertQuery("abc AND andsomething").resultsIn(WORD, AND, WORD);
-    Iterator<SearchQueryToken> it;
+    assertQuery("abc AND ANDsomething").resultsIn(WORD, AND, WORD);
-
+    assertQuery("abc ANDsomething").resultsIn(WORD, WORD);
-    result = tokenizer.tokenize("NOT abc AND nothing");
+    assertQuery("abc ORsomething").resultsIn(WORD, WORD);
-    
+    assertQuery("abc OR orsomething").resultsIn(WORD, OR, WORD);
-    it = result.iterator();
+    assertQuery("abc OR ORsomething").resultsIn(WORD, OR, WORD);
-    Assert.assertEquals(NOT, it.next().getToken());
-    Assert.assertEquals(WORD, it.next().getToken());
-    Assert.assertEquals(AND, it.next().getToken());
-    Assert.assertEquals(WORD, it.next().getToken());
-
-    result = tokenizer.tokenize("abc AND andsomething");
-    
-    it = result.iterator();
-    Assert.assertEquals(WORD, it.next().getToken());
-    Assert.assertEquals(AND, it.next().getToken());
-    Assert.assertEquals(WORD, it.next().getToken());
-
-    TokenizerValidator.init("abc AND ANDsomething")
-        .addExpected(WORD, AND, WORD).validate();
-
-    TokenizerValidator.init("abc ANDsomething")
-        .addExpected(WORD, WORD).validate();
-
-    TokenizerValidator.init("abc ORsomething")
-        .addExpected(WORD, WORD).validate();
-
-    TokenizerValidator.init("abc OR orsomething")
-        .addExpected(WORD, OR, WORD).validate();
-
-    TokenizerValidator.init("abc OR ORsomething")
-        .addExpected(WORD, OR, WORD).validate();
   }
 
   @Ignore
   @Test
   public void unicodeInWords() throws Exception {
     // Ll, Lm, Lo, Lt, Lu, Nl
-    TokenizerValidator.init("abc OR Ll\u01E3Lm\u02B5Lo\u1BE4Lt\u01F2Lu\u03D3Nl\u216F")
+    assertQuery("abc OR Ll\u01E3Lm\u02B5Lo\u1BE4Lt\u01F2Lu\u03D3Nl\u216F")
-        .addExpected(WORD, OR, WORD).validate();
+        .resultsIn(WORD, OR, WORD);
   }
 
   /**
@@ -369,8 +249,7 @@ public class SearchTokenizerTest {
    */
   @Test
   public void characterInPhrase() throws Exception {
-    TokenizerValidator.init("\"123\" OR \"ALPHA-._~\"")
+    assertQuery("\"123\" OR \"ALPHA-._~\"").resultsIn(PHRASE, OR, PHRASE);
-        .addExpected(PHRASE, OR, PHRASE).validate();
   }
 
   @Test
@@ -395,7 +274,7 @@ public class SearchTokenizerTest {
     validate("abc     def     ghi");
 
     // mixed not
-    TokenizerValidator.init("    abc         def AND     ghi").validate(WORD, WORD, AND, WORD);
+    assertQuery("    abc         def AND     ghi").resultsIn(WORD, WORD, AND, WORD);
     validate("NOT abc  NOT    def  OR NOT ghi", NOT, WORD, NOT, WORD, OR, NOT, WORD);
     validate("    abc         def     NOT ghi", WORD, WORD, NOT, WORD);
 
@@ -411,48 +290,60 @@ public class SearchTokenizerTest {
   @Test
   public void tokenizeInvalid() throws SearchTokenizerException {
     //
-    TokenizerValidator.init("(  abc AND) OR something").validate(SearchTokenizerException.class);
+    assertQuery("(  abc AND) OR something").resultsIn(SearchTokenizerException.class);
 
-    TokenizerValidator.init("\"phrase\"word").validate(SearchTokenizerException.class);
+    assertQuery("\"phrase\"word").resultsIn(SearchTokenizerException.class);
-    TokenizerValidator.init("\"p\"w").validate(SearchTokenizerException.class);
+    assertQuery("\"p\"w").resultsIn(SearchTokenizerException.class);
-    TokenizerValidator.init("\"\"").validate(SearchTokenizerException.class);
+    assertQuery("\"\"").resultsIn(SearchTokenizerException.class);
   }
 
   @Test
   public void tokenizeInvalidQueryForParser() throws SearchTokenizerException {
-    TokenizerValidator.init("AND").validate(AND);
+    assertQuery("AND").resultsIn(AND);
-    TokenizerValidator.init("OR").validate(OR);
+    assertQuery("OR").resultsIn(OR);
-    TokenizerValidator.init("NOT").validate(NOT);
+    assertQuery("NOT").resultsIn(NOT);
-    TokenizerValidator.init("NOT AND").validate(NOT, AND);
+    assertQuery("NOT AND").resultsIn(NOT, AND);
-    TokenizerValidator.init("NOT OR").validate(NOT, OR);
+    assertQuery("NOT OR").resultsIn(NOT, OR);
-    TokenizerValidator.init("NOT NOT").validate(NOT, NOT);
+    assertQuery("NOT NOT").resultsIn(NOT, NOT);
-    TokenizerValidator.init("abc AND OR something").validate(WORD, AND, OR, WORD);
+    assertQuery("abc AND OR something").resultsIn(WORD, AND, OR, WORD);
-    TokenizerValidator.init("abc AND \"something\" )").validate(WORD, AND, PHRASE, CLOSE);
+    assertQuery("abc AND \"something\" )").resultsIn(WORD, AND, PHRASE, CLOSE);
   }
 
   public void validate(String query) throws SearchTokenizerException {
-    new TokenizerValidator(query).validate();
+    new Validator(query);
+  }
+
+  public Validator assertQuery(String query) throws SearchTokenizerException {
+    return new Validator(query);
   }
 
   public void validate(String query, SearchQueryToken.Token ... tokens) throws SearchTokenizerException {
-    TokenizerValidator sv = new TokenizerValidator(query);
+    Validator sv = new Validator(query);
     for (SearchQueryToken.Token token : tokens) {
       sv.addExpected(token);
     }
     sv.validate();
   }
 
-  private static class TokenizerValidator {
+  private static class Validator {
     private List<Tuple> validations = new ArrayList<Tuple>();
     private boolean log;
     private final String searchQuery;
 
-    public void validate(SearchQueryToken.Token... tokens) throws SearchTokenizerException {
+    public void resultsIn(SearchQueryToken.Token... tokens) throws SearchTokenizerException {
       addExpected(tokens);
       validate();
     }
-
+    public void resultsIn(Tuple... tuple) throws SearchTokenizerException {
-    private class Tuple {
+      for (Tuple t : tuple) {
+        addExpected(t.token, t.literal);
+      }
+      validate();
+    }
+    public static Tuple tuple(SearchQueryToken.Token token, String literal) {
+      return new Tuple(token, literal);
+    }
+    private static class Tuple {
       final SearchQueryToken.Token token;
       final String literal;
       public Tuple(SearchQueryToken.Token token, String literal) {
@@ -464,30 +355,30 @@ public class SearchTokenizerTest {
       }
     }
 
-    private TokenizerValidator(String searchQuery) {
+    private Validator(String searchQuery) {
       this.searchQuery = searchQuery;
     }
 
-    private static TokenizerValidator init(String searchQuery) {
+    private static Validator init(String searchQuery) {
-      return new TokenizerValidator(searchQuery);
+      return new Validator(searchQuery);
     }
     
     @SuppressWarnings("unused")
-    private TokenizerValidator enableLogging() {
+    private Validator enableLogging() {
       log = true;
       return this;
     }
-    private TokenizerValidator addExpected(SearchQueryToken.Token token, String literal) {
+    private Validator addExpected(SearchQueryToken.Token token, String literal) {
       validations.add(new Tuple(token, literal));
       return this;
     }
-    private TokenizerValidator addExpected(SearchQueryToken.Token ... token) {
+    private Validator addExpected(SearchQueryToken.Token ... token) {
       for (SearchQueryToken.Token t : token) {
         validations.add(new Tuple(t));
       }
       return this;
     }
-    private void validate(Class<? extends Exception> exception) throws SearchTokenizerException {
+    private void resultsIn(Class<? extends Exception> exception) throws SearchTokenizerException {
       try {
         new SearchTokenizer().tokenize(searchQuery);
       } catch (Exception e) {
@@ -497,6 +388,17 @@ public class SearchTokenizerTest {
       Assert.fail("Expected exception " + exception.getClass().getSimpleName() + " was not thrown.");
     }
 
+    private void resultsIn(SearchTokenizerException.MessageKey key)
+        throws SearchTokenizerException {
+      try {
+        init(searchQuery).validate();
+      } catch (SearchTokenizerException e) {
+        Assert.assertEquals("SearchTokenizerException with unexpected message was thrown.", key, e.getMessageKey());
+        return;
+      }
+      Assert.fail("No SearchTokenizerException was not thrown.");
+    }
+
     private void validate() throws SearchTokenizerException {
       SearchTokenizer tokenizer = new SearchTokenizer();
       List<SearchQueryToken> result = tokenizer.tokenize(searchQuery);