[OLINGO-568] Improved states

lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java

@@ -82,6 +82,10 @@ public class SearchTokenizer {
       return token;
     }
 
+    public boolean close() {
+      return nextChar(EOF).isFinished();
+    }
+
     static boolean isAllowedChar(final char character) {
       // TODO mibo: add missing allowed characters
       return CHAR_A <= character && character <= 'Z' // case A..Z
@@ -173,11 +177,9 @@ public class SearchTokenizer {
       if (c == CHAR_OPEN) {
         return new OpenState();
       } else if (isWhitespace(c)) {
-        return new RwsState();
+        return new RwsImplicitAndState();
       } else if(c == CHAR_CLOSE) {
         return new CloseState();
-      } else if(isEof(c)) {
-        return finish();
       } else if(isWhitespace(c)) {
         return new AndState(c);
       } else {
@@ -185,6 +187,11 @@ public class SearchTokenizer {
       }
     }
 
+    @Override
+    public boolean close() {
+      return true;
+    }
+
     @Override
     public State init(char c) {
       return nextChar(c);
@@ -229,7 +236,7 @@ public class SearchTokenizer {
         return new CloseState();
       } else if (isWhitespace(c)) {
         finish();
-        return new RwsState();
+        return new RwsImplicitAndState();
       } else if (isEof(c)) {
         return finish();
       }
@@ -253,10 +260,16 @@ public class SearchTokenizer {
         return allowed(c);
       } else if (c == QUOTATION_MARK) {
         finish();
-        return allowed(c);
+        allowed(c);
+        return new SearchExpressionState();
       } else if (isWhitespace(c)) {
         if(isFinished()) {
-          return new RwsState();
+          return new RwsImplicitAndState();
+        }
+        return allowed(c);
+      } else if (c == CHAR_CLOSE) {
+        if(isFinished()) {
+          return new CloseState();
         }
         return allowed(c);
       } else if (isEof(c)) {
@@ -319,22 +332,6 @@ public class SearchTokenizer {
     }
   }
 
-  private class ImplicitAndState extends LiteralState {
-    private State followingState;
-    public ImplicitAndState(char c) {
-      super(Token.AND);
-      finish();
-      followingState = new SearchExpressionState().init(c);
-    }
-    public State nextState() {
-      return followingState;
-    }
-    @Override
-    public State nextChar(char c) {
-      return followingState.nextChar(c);
-    }
-  }
-
   private class AndState extends LiteralState {
     public AndState(char c) {
       super(Token.AND, c);
@@ -393,9 +390,10 @@ public class SearchTokenizer {
     }
   }
 
-  private class RwsState extends State {
+  // implicit and
-    public RwsState() {
+  private class RwsImplicitAndState extends State {
-      super(Token.RWS);
+    public RwsImplicitAndState() {
+      super(Token.AND);
     }
     @Override
     public State nextChar(char c) {
@@ -406,7 +404,8 @@ public class SearchTokenizer {
       } else if (c == CHAR_A) {
         return new AndState(c);
       } else {
-        return new ImplicitAndState(c);
+        finish();
+        return new SearchExpressionState().init(c);
       }
     }
   }
@@ -417,25 +416,18 @@ public class SearchTokenizer {
 
     State state = new SearchExpressionState();
     List<SearchQueryToken> states = new ArrayList<SearchQueryToken>();
-    State lastAdded = null;
     for (char aChar : chars) {
       State next = state.nextChar(aChar);
-      if (next instanceof ImplicitAndState) {
+      if (state.isFinished()) {
-        lastAdded = next;
-        states.add(next);
-        next = ((ImplicitAndState)next).nextState();
-      } else if (state.isFinished() && state != lastAdded) {
-        lastAdded = state;
         states.add(state);
       }
       state = next;
     }
 
-    final State lastState = state.nextChar(State.EOF);
+    if(state.close()) {
-    if(lastState.isFinished()) {
+     if(state.isFinished()) {
-      if(state != lastAdded) {
+       states.add(state);
-        states.add(state);
+     }
-      }
     } else {
       throw new IllegalStateException("State: " + state + " not finished and list is: " + states.toString());
     }

lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java

@@ -86,6 +86,8 @@ public class SearchTokenizerTest {
     SearchTokenizer tokenizer = new SearchTokenizer();
     List<SearchQueryToken> result;
 
+    SearchValidator.init("abc AND \"x-y_z\" AND 123").validate();
+
     //
     result = tokenizer.tokenize("\"abc\"");
     Assert.assertNotNull(result);
@@ -325,12 +327,17 @@ public class SearchTokenizerTest {
     // parenthesis
     validate("(abc)");
     validate("(abc AND  def)");
-    validate("(abc AND  def)   OR  ghi ");
+    validate("(abc AND  def)   OR  ghi");
-    validate("(abc AND  def)       ghi ");
+    validate("(abc AND  def)       ghi");
     validate("abc AND (def    OR  ghi)");
     validate("abc AND (def        ghi)");
   }
 
+  @Test
+  public void parseInvalid() {
+    SearchValidator.init("abc AND OR something").validate();
+  }
+
   public boolean validate(String query) {
     return new SearchValidator(query).validate();
   }