[OLINGO-568] Removed not necessary code and enums

lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchQueryToken.java

@@ -19,7 +19,7 @@
 package org.apache.olingo.server.core.uri.parser.search;
 
 public interface SearchQueryToken {
-  enum Token {OPEN, BWS, RWS, TERM, SEARCH_EXPRESSION, NOT, AND, OR, WORD, PHRASE, CLOSE}
+  enum Token {OPEN, TERM, NOT, AND, OR, WORD, PHRASE, CLOSE}
 
   Token getToken();
   String getLiteral();

lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java

@@ -44,7 +44,6 @@ public class SearchTokenizer {
     private Token token = null;
     private boolean finished = false;
 
-    protected static final char EOF = 0x03;
     protected static final char QUOTATION_MARK = '\"';
     protected static final char CHAR_N = 'N';
     protected static final char CHAR_O = 'O';
@@ -82,8 +81,8 @@ public class SearchTokenizer {
       return token;
     }
 
-    public boolean close() {
+    public State close() {
-      return nextChar(EOF).isFinished();
+      return this;
     }
 
     static boolean isAllowedChar(final char character) {
@@ -114,10 +113,6 @@ public class SearchTokenizer {
           || character == '=';
     }
 
-    static boolean isEof(final char character) {
-      return character == EOF;
-    }
-
     //BWS =  *( SP / HTAB / "%20" / "%09" )  ; "bad" whitespace
     //RWS = 1*( SP / HTAB / "%20" / "%09" )  ; "required" whitespace
     static boolean isWhitespace(final char character) {
@@ -138,7 +133,7 @@ public class SearchTokenizer {
   }
 
   private static abstract class LiteralState extends State {
-    private final StringBuilder literal = new StringBuilder();
+    protected final StringBuilder literal = new StringBuilder();
     public LiteralState(Token t) {
       super(t);
     }
@@ -170,7 +165,7 @@ public class SearchTokenizer {
 
   private class SearchExpressionState extends LiteralState {
     public SearchExpressionState() {
-      super(Token.SEARCH_EXPRESSION);
+      super(null);
     }
     @Override
     public State nextChar(char c) {
@@ -180,18 +175,11 @@ public class SearchTokenizer {
         return new RwsImplicitAndState();
       } else if(c == CHAR_CLOSE) {
         return new CloseState();
-      } else if(isWhitespace(c)) {
-        return new AndState(c);
       } else {
         return new SearchTermState().init(c);
       }
     }
 
-    @Override
-    public boolean close() {
-      return true;
-    }
-
     @Override
     public State init(char c) {
       return nextChar(c);
@@ -237,11 +225,14 @@ public class SearchTokenizer {
       } else if (isWhitespace(c)) {
         finish();
         return new RwsImplicitAndState();
-      } else if (isEof(c)) {
-        return finish();
       }
       return forbidden(c);
     }
+
+    @Override
+    public State close() {
+      return finish();
+    }
   }
 
   private class SearchPhraseState extends LiteralState {
@@ -254,7 +245,7 @@ public class SearchTokenizer {
 
     @Override
     public State nextChar(char c) {
-      if(isFinished() && !isEof(c)) {
+      if(isFinished()) {
         return new SearchExpressionState().init(c);
       } else if (isAllowedPhrase(c)) {
         return allowed(c);
@@ -272,8 +263,6 @@ public class SearchTokenizer {
           return new CloseState();
         }
         return allowed(c);
-      } else if (isEof(c)) {
-        return finish();
       }
       return forbidden(c);
     }
@@ -302,13 +291,9 @@ public class SearchTokenizer {
 
     @Override
     public State nextChar(char c) {
-      if (isEof(c)) {
-        return finish();
-      } else {
       return new SearchExpressionState().init(c);
     }
   }
-  }
 
   private class NotState extends LiteralState {
     public NotState(char c) {
@@ -319,9 +304,9 @@ public class SearchTokenizer {
     }
     @Override
     public State nextChar(char c) {
-      if (getLiteral().length() == 1 && (c == CHAR_O)) {
+      if (getLiteral().length() == 1 && c == CHAR_O) {
         return allowed(c);
-      } else if (getLiteral().length() == 2 && (c == CHAR_T)) {
+      } else if (getLiteral().length() == 2 && c == CHAR_T) {
         return allowed(c);
       } else if(getLiteral().length() == 3 && isWhitespace(c)) {
         finish();
@@ -341,11 +326,11 @@ public class SearchTokenizer {
     }
     @Override
     public State nextChar(char c) {
-      if (getLiteral().length() == 1 && (c == CHAR_N)) {
+      if (literal.length() == 1 && c == CHAR_N) {
         return allowed(c);
-      } else if (getLiteral().length() == 2 && (c == CHAR_D)) {
+      } else if (literal.length() == 2 && c == CHAR_D) {
         return allowed(c);
-      } else if(getLiteral().length() == 3 && isWhitespace(c)) {
+      } else if(literal.length() == 3 && isWhitespace(c)) {
         finish();
         return new BeforeSearchExpressionRwsState();
       } else {
@@ -363,9 +348,9 @@ public class SearchTokenizer {
     }
     @Override
     public State nextChar(char c) {
-      if (getLiteral().length() == 1 && (c == CHAR_R)) {
+      if (literal.length() == 1 && (c == CHAR_R)) {
         return allowed(c);
-      } else if(getLiteral().length() == 2 && isWhitespace(c)) {
+      } else if(literal.length() == 2 && isWhitespace(c)) {
         finish();
         return new BeforeSearchExpressionRwsState();
       } else {
@@ -378,7 +363,7 @@ public class SearchTokenizer {
   // RWS [ 'AND' RWS ] searchExpr
   private class BeforeSearchExpressionRwsState extends State {
     public BeforeSearchExpressionRwsState() {
-      super(Token.RWS);
+      super(null);
     }
     @Override
     public State nextChar(char c) {
@@ -424,13 +409,9 @@ public class SearchTokenizer {
       state = next;
     }
 
-    if(state.close()) {
+    if(state.close().isFinished()) {
-     if(state.isFinished()) {
       states.add(state);
     }
-    } else {
-      throw new IllegalStateException("State: " + state + " not finished and list is: " + states.toString());
-    }
 
     return states;
   }

lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java

@@ -287,15 +287,22 @@ public class SearchTokenizerTest {
     Assert.assertEquals(AND, it.next().getToken());
     Assert.assertEquals(WORD, it.next().getToken());
 
-    result = tokenizer.tokenize("abc OR orsomething");
-    log(result.toString());
-    it = result.iterator();
-    Assert.assertEquals(WORD, it.next().getToken());
-    Assert.assertEquals(OR, it.next().getToken());
-    Assert.assertEquals(WORD, it.next().getToken());
-
     SearchValidator.init("abc AND ANDsomething")
         .addExpected(WORD, AND, WORD).validate();
+
+    // FIXME (mibo): issue with implicit and
+//    SearchValidator.init("abc ANDsomething").enableLogging()
+//        .addExpected(WORD, AND, WORD).validate();
+
+//    SearchValidator.init("abc ORsomething")
+//        .addExpected(WORD, AND, WORD).validate();
+
+    SearchValidator.init("abc OR orsomething")
+        .addExpected(WORD, OR, WORD).validate();
+
+    SearchValidator.init("abc OR ORsomething")
+        .addExpected(WORD, OR, WORD).validate();
+
   }
 
   @Test