oops, forgot to check in JavaCC generated files

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150073 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java

@@ -47,6 +47,9 @@ public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer impl
     case NUM:
       token = jj_consume_token(NUM);
       break;
+    case SIGRAM:
+      token = jj_consume_token(SIGRAM);
+      break;
     case 0:
       token = jj_consume_token(0);
       break;
@@ -76,7 +79,7 @@ public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer impl
       jj_la1_0();
    }
    private static void jj_la1_0() {
-      jj_la1_0 = new int[] {0xff,};
+      jj_la1_0 = new int[] {0x4ff,};
    }
 
   public StandardTokenizer(CharStream stream) {
@@ -155,8 +158,8 @@ public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer impl
 
   public ParseException generateParseException() {
     jj_expentries.removeAllElements();
-    boolean[] la1tokens = new boolean[14];
-    for (int i = 0; i < 14; i++) {
+    boolean[] la1tokens = new boolean[16];
+    for (int i = 0; i < 16; i++) {
       la1tokens[i] = false;
     }
     if (jj_kind >= 0) {
@@ -172,7 +175,7 @@ public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer impl
         }
       }
     }
-    for (int i = 0; i < 14; i++) {
+    for (int i = 0; i < 16; i++) {
       if (la1tokens[i]) {
         jj_expentry = new int[1];
         jj_expentry[0] = i;

src/java/org/apache/lucene/analysis/standard/StandardTokenizerConstants.java

@@ -13,10 +13,12 @@ public interface StandardTokenizerConstants {
   int NUM = 7;
   int P = 8;
   int HAS_DIGIT = 9;
-  int ALPHA = 10;
-  int LETTER = 11;
-  int DIGIT = 12;
-  int NOISE = 13;
+  int SIGRAM = 10;
+  int ALPHA = 11;
+  int LETTER = 12;
+  int CJK = 13;
+  int DIGIT = 14;
+  int NOISE = 15;
 
   int DEFAULT = 0;
 
@@ -31,8 +33,10 @@ public interface StandardTokenizerConstants {
     "<NUM>",
     "<P>",
     "<HAS_DIGIT>",
+    "<SIGRAM>",
     "<ALPHA>",
     "<LETTER>",
+    "<CJK>",
     "<DIGIT>",
     "<NOISE>",
   };