SOLR-13437: fork noggit code into Solr (#666)

* SOLR-13437: fork noggit code into Solr

lucene/ivy-versions.properties

@@ -274,7 +274,6 @@ org.gagravarr.vorbis.java.version = 0.8
 
 /org.mockito/mockito-core = 2.23.4
 
-/org.noggit/noggit = 0.8
 /org.objenesis/objenesis = 2.6
 
 org.ow2.asm.version = 5.1

solr/CHANGES.txt

@@ -94,6 +94,8 @@ Other Changes
 
 * SOLR-13453: Adjust auth metrics asserts in tests caused by SOLR-13449 (janhoy)
 
+* SOLR-13437: noggit json parser is forked into solrj (noble)
+
 ==================  8.1.0 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

solr/NOTICE.txt

@@ -590,4 +590,17 @@ The Base64 implementation in this software was derived from the
 Apache Commons Codec project. http://commons.apache.org/proper/commons-codec/
 
 JSON processing in this software was derived from the JSON.simple toolkit.
-https://code.google.com/p/json-simple/
+https://code.google.com/p/json-simple/
+
+=========================================================================
+==         noggit notice                                               ==
+=========================================================================
+
+noggit
+
+Copyright 2006- Yonik Seeley
+
+Noggit is a  fast streaming JSON parser for java. The code is included
+into Solr codebase.
+
+https://github.com/yonik/noggit

solr/core/src/test/org/apache/solr/schema/TestSchemalessBufferedUpdates.java

@@ -145,7 +145,7 @@ public class TestSchemalessBufferedUpdates extends SolrTestCaseJ4 {
       UpdateRequestProcessor processor = chainUpToDUP.createProcessor(req, rsp);
       processor.processAdd(cmd);
       if (cmd.solrDoc.get("f_dt").getValue() instanceof Date) {
-        // Non-JSON types (Date in this case) aren't handled properly in noggit-0.6.  Although this is fixed in 
+        // Non-JSON types (Date in this case) aren't handled properly in noggit-0.6.  Although this is fixed in
         // https://github.com/yonik/noggit/commit/ec3e732af7c9425e8f40297463cbe294154682b1 to call obj.toString(), 
         // Date::toString produces a Date representation that Solr doesn't like, so we convert using Instant::toString
         cmd.solrDoc.get("f_dt").setValue(((Date) cmd.solrDoc.get("f_dt").getValue()).toInstant().toString());

solr/solrj/ivy.xml

@@ -36,7 +36,6 @@
     <dependency org="org.apache.commons" name="commons-math3" rev="${/org.apache.commons/commons-math3}" conf="compile"/>
     <dependency org="org.codehaus.woodstox" name="woodstox-core-asl" rev="${/org.codehaus.woodstox/woodstox-core-asl}" conf="compile"/>
     <dependency org="org.codehaus.woodstox" name="stax2-api" rev="${/org.codehaus.woodstox/stax2-api}" conf="compile"/>
-    <dependency org="org.noggit" name="noggit" rev="${/org.noggit/noggit}" conf="compile"/>
     <dependency org="org.slf4j" name="slf4j-api" rev="${/org.slf4j/slf4j-api}" conf="compile"/>
     <dependency org="org.slf4j" name="jcl-over-slf4j" rev="${/org.slf4j/jcl-over-slf4j}" conf="compile"/>
 

solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java

@@ -20,7 +20,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
-import org.noggit.JSONUtil;
+import org.apache.solr.common.util.Utils;
 
 public class Replica extends ZkNodeProps {
   
@@ -183,6 +183,6 @@ public class Replica extends ZkNodeProps {
 
   @Override
   public String toString() {
-    return name + ':' + JSONUtil.toJSON(propMap, -1); // small enough, keep it on one line (i.e. no indent)
+    return name + ':' + Utils.toJSONString(propMap); // small enough, keep it on one line (i.e. no indent)
   }
 }

solr/solrj/src/java/org/apache/solr/common/cloud/RoutingRule.java

@@ -21,7 +21,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.solr.common.util.SuppressForbidden;
-import org.noggit.JSONUtil;
+import org.apache.solr.common.util.Utils;
 
 /**
  * Used for routing docs with particular keys into another collection
@@ -72,6 +72,6 @@ public class RoutingRule extends ZkNodeProps {
 
   @Override
   public String toString() {
-    return JSONUtil.toJSON(propMap, -1);
+    return Utils.toJSONString(propMap);
   }
 }

solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java

@@ -21,6 +21,7 @@ import java.io.OutputStream;
 
 import org.noggit.CharArr;
 
+
 public class ByteUtils {
 
   /** Maximum number of UTF8 bytes per UTF16 character. */

solr/solrj/src/java/org/noggit/CharArr.java

@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.CharBuffer;
+
+public class CharArr implements CharSequence, Appendable {
+  protected char[] buf;
+  protected int start;
+  protected int end;
+
+  public CharArr() {
+    this(32);
+  }
+
+  public CharArr(int size) {
+    buf = new char[size];
+  }
+
+  public CharArr(char[] arr, int start, int end) {
+    set(arr, start, end);
+  }
+
+  public void setStart(int start) {
+    this.start = start;
+  }
+
+  public void setEnd(int end) {
+    this.end = end;
+  }
+
+  public void set(char[] arr, int start, int end) {
+    this.buf = arr;
+    this.start = start;
+    this.end = end;
+  }
+
+  public char[] getArray() {
+    return buf;
+  }
+
+  public int getStart() {
+    return start;
+  }
+
+  public int getEnd() {
+    return end;
+  }
+
+  public int size() {
+    return end - start;
+  }
+
+  @Override
+  public int length() {
+    return size();
+  }
+
+  /**
+   * The capacity of the buffer when empty (getArray().size())
+   */
+  public int capacity() {
+    return buf.length;
+  }
+
+
+  @Override
+  public char charAt(int index) {
+    return buf[start + index];
+  }
+
+  @Override
+  public CharArr subSequence(int start, int end) {
+    return new CharArr(buf, this.start + start, this.start + end);
+  }
+
+  public int read() throws IOException {
+    if (start >= end) return -1;
+    return buf[start++];
+  }
+
+  public int read(char cbuf[], int off, int len) {
+    //TODO
+    return 0;
+  }
+
+  public void unsafeWrite(char b) {
+    buf[end++] = b;
+  }
+
+  public void unsafeWrite(int b) {
+    unsafeWrite((char) b);
+  }
+
+  public void unsafeWrite(char b[], int off, int len) {
+    System.arraycopy(b, off, buf, end, len);
+    end += len;
+  }
+
+  protected void resize(int len) {
+    char newbuf[] = new char[Math.max(buf.length << 1, len)];
+    System.arraycopy(buf, start, newbuf, 0, size());
+    buf = newbuf;
+  }
+
+  public void reserve(int num) {
+    if (end + num > buf.length) resize(end + num);
+  }
+
+  public void write(char b) {
+    if (end >= buf.length) {
+      resize(end + 1);
+    }
+    unsafeWrite(b);
+  }
+
+  public final void write(int b) {
+    write((char) b);
+  }
+
+  public final void write(char[] b) {
+    write(b, 0, b.length);
+  }
+
+  public void write(char b[], int off, int len) {
+    reserve(len);
+    unsafeWrite(b, off, len);
+  }
+
+  public final void write(CharArr arr) {
+    write(arr.buf, arr.start, arr.end - arr.start);
+  }
+
+  public final void write(String s) {
+    write(s, 0, s.length());
+  }
+
+  public void write(String s, int stringOffset, int len) {
+    reserve(len);
+    s.getChars(stringOffset, len, buf, end);
+    end += len;
+  }
+
+  public void flush() {
+  }
+
+  public final void reset() {
+    start = end = 0;
+  }
+
+  public void close() {
+  }
+
+  public char[] toCharArray() {
+    char newbuf[] = new char[size()];
+    System.arraycopy(buf, start, newbuf, 0, size());
+    return newbuf;
+  }
+
+
+  @Override
+  public String toString() {
+    return new String(buf, start, size());
+  }
+
+  public int read(CharBuffer cb) throws IOException {
+
+    /***
+     int sz = size();
+     if (sz<=0) return -1;
+     if (sz>0) cb.put(buf, start, sz);
+     return -1;
+     ***/
+
+    int sz = size();
+    if (sz > 0) cb.put(buf, start, sz);
+    start = end;
+    while (true) {
+      fill();
+      int s = size();
+      if (s == 0) return sz == 0 ? -1 : sz;
+      sz += s;
+      cb.put(buf, start, s);
+    }
+  }
+
+
+  public int fill() throws IOException {
+    return 0;  // or -1?
+  }
+
+  //////////////// Appendable methods /////////////
+  @Override
+  public final Appendable append(CharSequence csq) throws IOException {
+    return append(csq, 0, csq.length());
+  }
+
+  @Override
+  public Appendable append(CharSequence csq, int start, int end) throws IOException {
+    write(csq.subSequence(start, end).toString());
+    return null;
+  }
+
+  @Override
+  public final Appendable append(char c) throws IOException {
+    write(c);
+    return this;
+  }
+}
+
+
+class NullCharArr extends CharArr {
+  public NullCharArr() {
+    super(new char[1], 0, 0);
+  }
+
+  @Override
+  public void unsafeWrite(char b) {
+  }
+
+  @Override
+  public void unsafeWrite(char b[], int off, int len) {
+  }
+
+  @Override
+  public void unsafeWrite(int b) {
+  }
+
+  @Override
+  public void write(char b) {
+  }
+
+  @Override
+  public void write(char b[], int off, int len) {
+  }
+
+  @Override
+  public void reserve(int num) {
+  }
+
+  @Override
+  protected void resize(int len) {
+  }
+
+  @Override
+  public Appendable append(CharSequence csq, int start, int end) throws IOException {
+    return this;
+  }
+
+  @Override
+  public char charAt(int index) {
+    return 0;
+  }
+
+  @Override
+  public void write(String s, int stringOffset, int len) {
+  }
+}
+
+
+// IDEA: a subclass that refills the array from a reader?
+class CharArrReader extends CharArr {
+  protected final Reader in;
+
+  public CharArrReader(Reader in, int size) {
+    super(size);
+    this.in = in;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (start >= end) fill();
+    return start >= end ? -1 : buf[start++];
+  }
+
+  @Override
+  public int read(CharBuffer cb) throws IOException {
+    // empty the buffer and then read direct
+    int sz = size();
+    if (sz > 0) cb.put(buf, start, end);
+    int sz2 = in.read(cb);
+    if (sz2 >= 0) return sz + sz2;
+    return sz > 0 ? sz : -1;
+  }
+
+  @Override
+  public int fill() throws IOException {
+    if (start >= end) {
+      reset();
+    } else if (start > 0) {
+      System.arraycopy(buf, start, buf, 0, size());
+      end = size();
+      start = 0;
+    }
+    /***
+     // fill fully or not???
+     do {
+     int sz = in.read(buf,end,buf.length-end);
+     if (sz==-1) return;
+     end+=sz;
+     } while (end < buf.length);
+     ***/
+
+    int sz = in.read(buf, end, buf.length - end);
+    if (sz > 0) end += sz;
+    return sz;
+  }
+
+}
+
+
+class CharArrWriter extends CharArr {
+  protected Writer sink;
+
+  @Override
+  public void flush() {
+    try {
+      sink.write(buf, start, end - start);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    start = end = 0;
+  }
+
+  @Override
+  public void write(char b) {
+    if (end >= buf.length) {
+      flush();
+    }
+    unsafeWrite(b);
+  }
+
+  @Override
+  public void write(char b[], int off, int len) {
+    int space = buf.length - end;
+    if (len < space) {
+      unsafeWrite(b, off, len);
+    } else if (len < buf.length) {
+      unsafeWrite(b, off, space);
+      flush();
+      unsafeWrite(b, off + space, len - space);
+    } else {
+      flush();
+      try {
+        sink.write(b, off, len);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  @Override
+  public void write(String s, int stringOffset, int len) {
+    int space = buf.length - end;
+    if (len < space) {
+      s.getChars(stringOffset, stringOffset + len, buf, end);
+      end += len;
+    } else if (len < buf.length) {
+      // if the data to write is small enough, buffer it.
+      s.getChars(stringOffset, stringOffset + space, buf, end);
+      flush();
+      s.getChars(stringOffset + space, stringOffset + len, buf, 0);
+      end = len - space;
+    } else {
+      flush();
+      // don't buffer, just write to sink
+      try {
+        sink.write(s, stringOffset, len);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+
+    }
+  }
+}

solr/solrj/src/java/org/noggit/JSONUtil.java

@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.noggit;
+
+
+public class JSONUtil {
+  public static final char[] TRUE_CHARS = new char[]{'t', 'r', 'u', 'e'};
+  public static final char[] FALSE_CHARS = new char[]{'f', 'a', 'l', 's', 'e'};
+  public static final char[] NULL_CHARS = new char[]{'n', 'u', 'l', 'l'};
+  public static final char[] HEX_CHARS = new char[]{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+  public static final char VALUE_SEPARATOR = ',';
+  public static final char NAME_SEPARATOR = ':';
+  public static final char OBJECT_START = '{';
+  public static final char OBJECT_END = '}';
+  public static final char ARRAY_START = '[';
+  public static final char ARRAY_END = ']';
+
+  public static String toJSON(Object o) {
+    CharArr out = new CharArr();
+    new JSONWriter(out).write(o);
+    return out.toString();
+  }
+
+  /**
+   * @param o          The object to convert to JSON
+   * @param indentSize The number of space characters to use as an indent (default 2). 0=newlines but no spaces, -1=no indent at all.
+   */
+  public static String toJSON(Object o, int indentSize) {
+    CharArr out = new CharArr();
+    new JSONWriter(out, indentSize).write(o);
+    return out.toString();
+  }
+
+  public static void writeNumber(int number, CharArr out) {
+    out.write(Integer.toString(number));
+  }
+
+  public static void writeNumber(long number, CharArr out) {
+    out.write(Long.toString(number));
+  }
+
+  public static void writeNumber(float number, CharArr out) {
+    out.write(Float.toString(number));
+  }
+
+  public static void writeNumber(double number, CharArr out) {
+    out.write(Double.toString(number));
+  }
+
+  public static void writeString(CharArr val, CharArr out) {
+    writeString(val.getArray(), val.getStart(), val.getEnd(), out);
+  }
+
+  public static void writeString(char[] val, int start, int end, CharArr out) {
+    out.write('"');
+    writeStringPart(val, start, end, out);
+    out.write('"');
+  }
+
+  public static void writeString(String val, int start, int end, CharArr out) {
+    out.write('"');
+    writeStringPart(val, start, end, out);
+    out.write('"');
+  }
+
+  public static void writeString(CharSequence val, int start, int end, CharArr out) {
+    out.write('"');
+    writeStringPart(val, start, end, out);
+    out.write('"');
+  }
+
+  public static void writeStringPart(char[] val, int start, int end, CharArr out) {
+    for (int i = start; i < end; i++) {
+      char ch = val[i];
+      // When ch>=1f, (ch*146087937)&0xd6a01f80) is 0 only for characters that need escaping: " \\ u2028 u2029
+      // and has 7 false positives: 204a 4051 802f c022 c044 e04a e04b
+      if (ch > 0x1f && ((ch * 146087937) & 0xd6a01f80) != 0) {
+        out.write(ch);
+      } else {
+        writeChar(ch, out);
+      }
+    }
+  }
+
+  public static void writeChar(char ch, CharArr out) {
+    switch (ch) {
+      case '"':
+      case '\\':
+        out.write('\\');
+        out.write(ch);
+        break;
+      case '\r':
+        out.write('\\');
+        out.write('r');
+        break;
+      case '\n':
+        out.write('\\');
+        out.write('n');
+        break;
+      case '\t':
+        out.write('\\');
+        out.write('t');
+        break;
+      case '\b':
+        out.write('\\');
+        out.write('b');
+        break;
+      case '\f':
+        out.write('\\');
+        out.write('f');
+        break;
+      // case '/':
+      case '\u2028':  // valid JSON, but not valid json script
+      case '\u2029':
+        unicodeEscape(ch, out);
+        break;
+      default:
+        if (ch <= 0x1F) {
+          unicodeEscape(ch, out);
+        } else {
+          out.write(ch);
+        }
+    }
+  }
+
+
+  public static void writeStringPart(String chars, int start, int end, CharArr out) {
+    // TODO: write in chunks?
+
+    int toWrite = end - start;
+    char[] arr = out.getArray();
+    int pos = out.getEnd();
+    int space = arr.length - pos;
+    if (space < toWrite) {
+      writeStringPart((CharSequence) chars, start, end, out);
+      return;
+    }
+
+    // get chars directly from String into output array
+    chars.getChars(start, end, arr, pos);
+
+    int endInOut = pos + toWrite;
+    out.setEnd(endInOut);
+    for (int i = pos; i < endInOut; i++) {
+      char ch = arr[i];
+
+      // When ch>=1f, (ch*146087937)&0xd6a01f80) is 0 only for characters that need escaping: " \\ u2028 u2029
+      // and has 7 false positives: 204a 4051 802f c022 c044 e04a e04b
+      if (ch <= 0x1f || ((ch * 146087937) & 0xd6a01f80) == 0) {
+        // We hit a char that needs escaping. do the rest char by char.
+        out.setEnd(i);
+        writeStringPart((CharSequence) chars, start + (i - pos), end, out);
+        return;
+      }
+    }
+  }
+
+  public static void writeStringPart(CharSequence chars, int start, int end, CharArr out) {
+    for (int i = start; i < end; i++) {
+      char ch = chars.charAt(i);
+      // When ch>=1f, (ch*146087937)&0xd6a01f80) is 0 only for characters that need escaping: " \\ u2028 u2029
+      // and has 7 false positives: 204a 4051 802f c022 c044 e04a e04b
+      if (ch > 0x1f && ((ch * 146087937) & 0xd6a01f80) != 0) {
+        out.write(ch);
+      } else {
+        writeChar(ch, out);
+      }
+    }
+  }
+
+
+  public static void unicodeEscape(int ch, CharArr out) {
+    out.write('\\');
+    out.write('u');
+    out.write(HEX_CHARS[ch >>> 12]);
+    out.write(HEX_CHARS[(ch >>> 8) & 0xf]);
+    out.write(HEX_CHARS[(ch >>> 4) & 0xf]);
+    out.write(HEX_CHARS[ch & 0xf]);
+  }
+
+  public static void writeNull(CharArr out) {
+    out.write(NULL_CHARS);
+  }
+
+  public static void writeBoolean(boolean val, CharArr out) {
+    out.write(val ? TRUE_CHARS : FALSE_CHARS);
+  }
+
+}

solr/solrj/src/java/org/noggit/JSONWriter.java

@@ -0,0 +1,358 @@
+/*
+ *  Copyright 2006- Yonik Seeley
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.util.*;
+
+
+public class JSONWriter {
+
+  /**
+   * Implement this interface on your class to support serialization
+   */
+  public static interface Writable {
+    public void write(JSONWriter writer);
+  }
+
+  protected int level;
+  protected int indent;
+  protected final CharArr out;
+
+  /**
+   * @param out        the CharArr to write the output to.
+   * @param indentSize The number of space characters to use as an indent (default 2). 0=newlines but no spaces, -1=no indent at all.
+   */
+  public JSONWriter(CharArr out, int indentSize) {
+    this.out = out;
+    this.indent = indentSize;
+  }
+
+  public JSONWriter(CharArr out) {
+    this(out, 2);
+  }
+
+  public void setIndentSize(int indentSize) {
+    this.indent = indentSize;
+  }
+
+  public void indent() {
+    if (indent >= 0) {
+      out.write('\n');
+      if (indent > 0) {
+        int spaces = level * indent;
+        out.reserve(spaces);
+        for (int i = 0; i < spaces; i++) {
+          out.unsafeWrite(' ');
+        }
+      }
+    }
+  }
+
+  public void write(Object o) {
+    // NOTE: an instance-of chain was about 50% faster than hashing on the classes, even with perfect hashing.
+    if (o == null) {
+      writeNull();
+    } else if (o instanceof String) {
+      writeString((String) o);
+    } else if (o instanceof Number) {
+      if (o instanceof Integer || o instanceof Long) {
+        write(((Number) o).longValue());
+      } else if (o instanceof Float || o instanceof Double) {
+        write(((Number) o).doubleValue());
+      } else {
+        CharArr arr = new CharArr();
+        arr.write(o.toString());
+        writeNumber(arr);
+      }
+    } else if (o instanceof Map) {
+      write((Map<?, ?>) o);
+    } else if (o instanceof Collection) {
+      write((Collection<?>) o);
+    } else if (o instanceof Boolean) {
+      write(((Boolean) o).booleanValue());
+    } else if (o instanceof CharSequence) {
+      writeString((CharSequence) o);
+    } else if (o instanceof Writable) {
+      ((Writable) o).write(this);
+    } else if (o instanceof Object[]) {
+      write(Arrays.asList((Object[]) o));
+    } else if (o instanceof int[]) {
+      write((int[]) o);
+    } else if (o instanceof float[]) {
+      write((float[]) o);
+    } else if (o instanceof long[]) {
+      write((long[]) o);
+    } else if (o instanceof double[]) {
+      write((double[]) o);
+    } else if (o instanceof short[]) {
+      write((short[]) o);
+    } else if (o instanceof boolean[]) {
+      write((boolean[]) o);
+    } else if (o instanceof char[]) {
+      write((char[]) o);
+    } else if (o instanceof byte[]) {
+      write((byte[]) o);
+    } else {
+      handleUnknownClass(o);
+    }
+  }
+
+  /**
+   * Override this method for custom handling of unknown classes.  Also see the Writable interface.
+   */
+  public void handleUnknownClass(Object o) {
+    writeString(o.toString());
+  }
+
+  public void write(Map<?, ?> val) {
+    startObject();
+    int sz = val.size();
+    boolean first = true;
+    for (Map.Entry<?, ?> entry : val.entrySet()) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      if (sz > 1) indent();
+      writeString(entry.getKey().toString());
+      writeNameSeparator();
+      write(entry.getValue());
+    }
+    endObject();
+  }
+
+  public void write(Collection<?> val) {
+    startArray();
+    int sz = val.size();
+    boolean first = true;
+    for (Object o : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      if (sz > 1) indent();
+      write(o);
+    }
+    endArray();
+  }
+
+  /**
+   * A byte[] may be either a single logical value, or a list of small integers.
+   * It's up to the implementation to decide.
+   */
+  public void write(byte[] val) {
+    startArray();
+    boolean first = true;
+    for (short v : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      write(v);
+    }
+    endArray();
+  }
+
+  public void write(short[] val) {
+    startArray();
+    boolean first = true;
+    for (short v : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      write(v);
+    }
+    endArray();
+  }
+
+  public void write(int[] val) {
+    startArray();
+    boolean first = true;
+    for (int v : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      write(v);
+    }
+    endArray();
+  }
+
+  public void write(long[] val) {
+    startArray();
+    boolean first = true;
+    for (long v : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      write(v);
+    }
+    endArray();
+  }
+
+  public void write(float[] val) {
+    startArray();
+    boolean first = true;
+    for (float v : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      write(v);
+    }
+    endArray();
+  }
+
+  public void write(double[] val) {
+    startArray();
+    boolean first = true;
+    for (double v : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      write(v);
+    }
+    endArray();
+  }
+
+  public void write(boolean[] val) {
+    startArray();
+    boolean first = true;
+    for (boolean v : val) {
+      if (first) {
+        first = false;
+      } else {
+        writeValueSeparator();
+      }
+      write(v);
+    }
+    endArray();
+  }
+
+
+  public void write(short number) {
+    write((int) number);
+  }
+
+  public void write(byte number) {
+    write((int) number);
+  }
+
+
+  public void writeNull() {
+    JSONUtil.writeNull(out);
+  }
+
+  public void writeString(String str) {
+    JSONUtil.writeString(str, 0, str.length(), out);
+  }
+
+  public void writeString(CharSequence str) {
+    JSONUtil.writeString(str, 0, str.length(), out);
+  }
+
+  public void writeString(CharArr str) {
+    JSONUtil.writeString(str, out);
+  }
+
+  public void writeStringStart() {
+    out.write('"');
+  }
+
+  public void writeStringChars(CharArr partialStr) {
+    JSONUtil.writeStringPart(partialStr.getArray(), partialStr.getStart(), partialStr.getEnd(), out);
+  }
+
+  public void writeStringEnd() {
+    out.write('"');
+  }
+
+  public void write(long number) {
+    JSONUtil.writeNumber(number, out);
+  }
+
+  public void write(int number) {
+    JSONUtil.writeNumber(number, out);
+  }
+
+  public void write(double number) {
+    JSONUtil.writeNumber(number, out);
+  }
+
+  public void write(float number) {
+    JSONUtil.writeNumber(number, out);
+  }
+
+  public void write(boolean bool) {
+    JSONUtil.writeBoolean(bool, out);
+  }
+
+  public void write(char[] val) {
+    JSONUtil.writeString(val, 0, val.length, out);
+  }
+
+  public void writeNumber(CharArr digits) {
+    out.write(digits);
+  }
+
+  public void writePartialNumber(CharArr digits) {
+    out.write(digits);
+  }
+
+  public void startObject() {
+    out.write('{');
+    level++;
+  }
+
+  public void endObject() {
+    out.write('}');
+    level--;
+  }
+
+  public void startArray() {
+    out.write('[');
+    level++;
+  }
+
+  public void endArray() {
+    out.write(']');
+    level--;
+  }
+
+  public void writeValueSeparator() {
+    out.write(',');
+  }
+
+  public void writeNameSeparator() {
+    out.write(':');
+  }
+
+}

solr/solrj/src/java/org/noggit/ObjectBuilder.java

@@ -0,0 +1,168 @@
+/*
+ *  Copyright 2006- Yonik Seeley
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+
+import java.util.*;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+public class ObjectBuilder {
+
+  public static Object fromJSON(String json) throws IOException {
+    JSONParser p = new JSONParser(json);
+    return getVal(p);
+  }
+
+  public static Object getVal(JSONParser parser) throws IOException {
+    return new ObjectBuilder(parser).getVal();
+  }
+
+  final JSONParser parser;
+
+  public ObjectBuilder(JSONParser parser) throws IOException {
+    this.parser = parser;
+    if (parser.lastEvent() == 0) parser.nextEvent();
+  }
+
+
+  public Object getVal() throws IOException {
+    int ev = parser.lastEvent();
+    switch (ev) {
+      case JSONParser.STRING:
+        return getString();
+      case JSONParser.LONG:
+        return getLong();
+      case JSONParser.NUMBER:
+        return getNumber();
+      case JSONParser.BIGNUMBER:
+        return getBigNumber();
+      case JSONParser.BOOLEAN:
+        return getBoolean();
+      case JSONParser.NULL:
+        return getNull();
+      case JSONParser.OBJECT_START:
+        return getObject();
+      case JSONParser.OBJECT_END:
+        return null; // OR ERROR?
+      case JSONParser.ARRAY_START:
+        return getArray();
+      case JSONParser.ARRAY_END:
+        return null; // OR ERROR?
+      case JSONParser.EOF:
+        return null; // OR ERROR?
+      default:
+        return null; // OR ERROR?
+    }
+  }
+
+
+  public Object getString() throws IOException {
+    return parser.getString();
+  }
+
+  public Object getLong() throws IOException {
+    return Long.valueOf(parser.getLong());
+  }
+
+  public Object getNumber() throws IOException {
+    CharArr num = parser.getNumberChars();
+    String numstr = num.toString();
+    double d = Double.parseDouble(numstr);
+    if (!Double.isInfinite(d)) return Double.valueOf(d);
+    // TODO: use more efficient constructor in Java5
+    return new BigDecimal(num.buf, num.start, num.size());
+  }
+
+  public Object getBigNumber() throws IOException {
+    CharArr num = parser.getNumberChars();
+    String numstr = num.toString();
+    for (int ch; (ch = num.read()) != -1; ) {
+      if (ch == '.' || ch == 'e' || ch == 'E') return new BigDecimal(numstr);
+    }
+    return new BigInteger(numstr);
+  }
+
+  public Object getBoolean() throws IOException {
+    return parser.getBoolean();
+  }
+
+  public Object getNull() throws IOException {
+    parser.getNull();
+    return null;
+  }
+
+  public Object newObject() throws IOException {
+    return new LinkedHashMap<Object, Object>();
+  }
+
+  public Object getKey() throws IOException {
+    return parser.getString();
+  }
+
+  @SuppressWarnings("unchecked")
+  public void addKeyVal(Object map, Object key, Object val) throws IOException {
+    /* Object prev = */
+    ((Map<Object, Object>) map).put(key, val);
+    // TODO: test for repeated value?
+  }
+
+  public Object objectEnd(Object obj) {
+    return obj;
+  }
+
+
+  public Object getObject() throws IOException {
+    Object m = newObject();
+    for (; ; ) {
+      int ev = parser.nextEvent();
+      if (ev == JSONParser.OBJECT_END) return objectEnd(m);
+      Object key = getKey();
+      ev = parser.nextEvent();
+      Object val = getVal();
+      addKeyVal(m, key, val);
+    }
+  }
+
+  public Object newArray() {
+    return new ArrayList<Object>();
+  }
+
+  @SuppressWarnings("unchecked")
+  public void addArrayVal(Object arr, Object val) throws IOException {
+    ((List<Object>) arr).add(val);
+  }
+
+  public Object endArray(Object arr) {
+    return arr;
+  }
+
+  public Object getArray() throws IOException {
+    Object arr = newArray();
+    for (; ; ) {
+      int ev = parser.nextEvent();
+      if (ev == JSONParser.ARRAY_END) return endArray(arr);
+      Object val = getVal();
+      addArrayVal(arr, val);
+    }
+  }
+
+}

solr/solrj/src/test/org/noggit/TestJSONParser.java

@@ -0,0 +1,690 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+public class TestJSONParser extends SolrTestCaseJ4 {
+
+  // these are to aid in debugging if an unexpected error occurs
+  static int parserType;
+  static int bufferSize;
+  static String parserInput;
+  static JSONParser lastParser;
+
+  static int flags = JSONParser.FLAGS_DEFAULT;  // the default
+
+  public static String lastParser() {
+    return "parserType=" + parserType
+        + (parserType==1 ? " bufferSize=" + bufferSize : "")
+        + " parserInput='" + parserInput + "'" + "flags : " + lastParser.flags;
+  }
+
+  public static JSONParser getParser(String s) {
+    return getParser(s, random().nextInt(2), -1);
+  }
+
+  public static JSONParser getParser(String s, int type, int bufSize) {
+    parserInput = s;
+    parserType = type;
+
+    JSONParser parser=null;
+    switch (type) {
+      case 0:
+        // test directly using input buffer
+        parser = new JSONParser(s.toCharArray(),0,s.length());
+        break;
+      case 1:
+        // test using Reader...
+        // small input buffers can help find bugs on boundary conditions
+
+        if (bufSize < 1) bufSize = random().nextInt(25) + 1;
+        bufferSize = bufSize;// record in case there is an error
+        parser = new JSONParser(new StringReader(s), new char[bufSize]);
+        break;
+    }
+    if (parser == null) return null;
+
+    lastParser = parser;
+
+    if (flags != JSONParser.FLAGS_DEFAULT) {
+      parser.setFlags(flags);
+    }
+
+    return parser;
+  }
+
+  /** for debugging purposes
+   public void testSpecific() throws Exception {
+   JSONParser parser = getParser("[0",1,1);
+   for (;;) {
+   int ev = parser.nextEvent();
+   if (ev == JSONParser.EOF) {
+   break;
+   } else {
+   System.out.println("got " + JSONParser.getEventString(ev));
+   }
+   }
+   }
+   **/
+
+  public static byte[] events = new byte[256];
+  static {
+    events['{'] = JSONParser.OBJECT_START;
+    events['}'] = JSONParser.OBJECT_END;
+    events['['] = JSONParser.ARRAY_START;
+    events[']'] = JSONParser.ARRAY_END;
+    events['s'] = JSONParser.STRING;
+    events['b'] = JSONParser.BOOLEAN;
+    events['l'] = JSONParser.LONG;
+    events['n'] = JSONParser.NUMBER;
+    events['N'] = JSONParser.BIGNUMBER;
+    events['0'] = JSONParser.NULL;
+    events['e'] = JSONParser.EOF;
+  }
+
+  // match parser states with the expected states
+  public static void parse(JSONParser p, String input, String expected) throws IOException {
+    expected += "e";
+    for (int i=0; i<expected.length(); i++) {
+      int ev = p.nextEvent();
+      int expect = events[expected.charAt(i)];
+      if (ev != expect) {
+        fail("Expected " + expect + ", got " + ev
+            + "\n\tINPUT=" + input
+            + "\n\tEXPECTED=" + expected
+            + "\n\tAT=" + i + " (" + expected.charAt(i) + ")");
+      }
+    }
+  }
+
+  public static void parse(String input, String expected) throws IOException {
+    String in = input;
+    if ((flags & JSONParser.ALLOW_SINGLE_QUOTES)==0 || random().nextBoolean()) {
+      in = in.replace('\'', '"');
+    }
+
+    for (int i=0; i<Integer.MAX_VALUE; i++) {
+      JSONParser p = getParser(in,i,-1);
+      if (p==null) break;
+      parse(p,in,expected);
+    }
+
+    testCorruption(input, 100000);
+
+  }
+
+  public static void testCorruption(String input, int iter) {
+    char[] arr = new char[input.length()];
+
+    for (int i=0; i<iter; i++) {
+      input.getChars(0, arr.length, arr, 0);
+      int changes = random().nextInt(arr.length>>1) + 1;
+      for (int j=0; j<changes; j++) {
+        char ch;
+        switch (random().nextInt(31)) {
+          case 0: ch = 0; break;
+          case 1: ch = '['; break;
+          case 2: ch = ']'; break;
+          case 3: ch = '{'; break;
+          case 4: ch = '}'; break;
+          case 5: ch = '"'; break;
+          case 6: ch = '\''; break;
+          case 7: ch = ' '; break;
+          case 8: ch = '\r'; break;
+          case 9: ch = '\n'; break;
+          case 10:ch = '\t'; break;
+          case 11:ch = ','; break;
+          case 12:ch = ':'; break;
+          case 13:ch = '.'; break;
+          case 14:ch = 'a'; break;
+          case 15:ch = 'e'; break;
+          case 16:ch = '0'; break;
+          case 17:ch = '1'; break;
+          case 18:ch = '+'; break;
+          case 19:ch = '-'; break;
+          case 20:ch = 't'; break;
+          case 21:ch = 'f'; break;
+          case 22:ch = 'n'; break;
+          case 23:ch = '/'; break;
+          case 24:ch = '\\'; break;
+          case 25:ch = 'u'; break;
+          case 26:ch = '\u00a0'; break;
+          default:ch = (char) random().nextInt(256);
+        }
+
+        arr[random().nextInt(arr.length)] = ch;
+      }
+
+
+      JSONParser parser = getParser(new String(arr));
+      parser.setFlags( random().nextInt() );  // set random parser flags
+
+      int ret = 0;
+      try {
+        for (;;) {
+          int ev = parser.nextEvent();
+          if (random().nextBoolean()) {
+            // see if we can read the event
+            switch (ev) {
+              case JSONParser.STRING: ret += parser.getString().length(); break;
+              case JSONParser.BOOLEAN: ret += parser.getBoolean() ? 1 : 2; break;
+              case JSONParser.BIGNUMBER: ret += parser.getNumberChars().length(); break;
+              case JSONParser.NUMBER: ret += parser.getDouble(); break;
+              case JSONParser.LONG: ret += parser.getLong(); break;
+              default: ret += ev;
+            }
+          }
+
+          if (ev == JSONParser.EOF) break;
+        }
+      } catch (IOException ex) {
+        // shouldn't happen
+        System.out.println(ret);  // use ret
+      } catch (JSONParser.ParseException ex) {
+        // OK
+      } catch (Throwable ex) {
+        ex.printStackTrace();
+        System.out.println(lastParser());
+        throw new RuntimeException(ex);
+      }
+    }
+  }
+
+
+
+  public static class Num {
+    public String digits;
+    public Num(String digits) {
+      this.digits = digits;
+    }
+    @Override
+    public String toString() { return new String("NUMBERSTRING("+digits+")"); }
+    @Override
+    public boolean equals(Object o) {
+      return (getClass() == o.getClass() && digits.equals(((Num) o).digits));
+    }
+    @Override
+    public int hashCode() {
+      return digits.hashCode();
+    }
+  }
+
+  public static class BigNum extends Num {
+    @Override
+    public String toString() { return new String("BIGNUM("+digits+")"); }
+    public BigNum(String digits) { super(digits); }
+  }
+
+  // Oh, what I wouldn't give for Java5 varargs and autoboxing
+  public static Long o(int l) { return (long) l; }
+  public static Long o(long l) { return l; }
+  public static Double o(double d) { return d; }
+  public static Boolean o(boolean b) { return b; }
+  public static Num n(String digits) { return new Num(digits); }
+  public static Num bn(String digits) { return new BigNum(digits); }
+  public static Object t = Boolean.TRUE;
+  public static Object f = Boolean.FALSE;
+  public static Object a = new Object(){@Override
+  public String toString() {return "ARRAY_START";}};
+  public static Object A = new Object(){@Override
+  public String toString() {return "ARRAY_END";}};
+  public static Object m = new Object(){@Override
+  public String toString() {return "OBJECT_START";}};
+  public static Object M = new Object(){@Override
+  public String toString() {return "OBJECT_END";}};
+  public static Object N = new Object(){@Override
+  public String toString() {return "NULL";}};
+  public static Object e = new Object(){@Override
+  public String toString() {return "EOF";}};
+
+  // match parser states with the expected states
+  public static void parse(JSONParser p, String input, Object[] expected) throws IOException {
+    for (int i=0; i<expected.length; i++) {
+      int ev = p.nextEvent();
+      Object exp = expected[i];
+      Object got = null;
+
+      switch(ev) {
+        case JSONParser.ARRAY_START: got=a; break;
+        case JSONParser.ARRAY_END: got=A; break;
+        case JSONParser.OBJECT_START: got=m; break;
+        case JSONParser.OBJECT_END: got=M; break;
+        case JSONParser.LONG: got=o(p.getLong()); break;
+        case JSONParser.NUMBER:
+          if (exp instanceof Double) {
+            got = o(p.getDouble());
+          } else {
+            got = n(p.getNumberChars().toString());
+          }
+          break;
+        case JSONParser.BIGNUMBER: got=bn(p.getNumberChars().toString()); break;
+        case JSONParser.NULL: got=N; p.getNull(); break; // optional
+        case JSONParser.BOOLEAN: got=o(p.getBoolean()); break;
+        case JSONParser.EOF: got=e; break;
+        case JSONParser.STRING: got=p.getString(); break;
+        default: got="Unexpected Event Number " + ev;
+      }
+
+      if (!(exp==got || exp.equals(got))) {
+        fail("Fail: String='" + input + "'" + "\n\tINPUT=" + got + "\n\tEXPECTED=" + exp + "\n\tAT RULE " + i);
+      }
+    }
+  }
+
+
+  public static void parse(String input, Object[] expected) throws IOException {
+    parse(input, (flags & JSONParser.ALLOW_SINGLE_QUOTES)==0 || random().nextBoolean(), expected);
+  }
+
+  public static void parse(String input, boolean changeSingleQuote, Object[] expected) throws IOException {
+    String in = input;
+    if (changeSingleQuote) {
+      in = in.replace('\'', '"');
+    }
+    for (int i=0; i<Integer.MAX_VALUE; i++) {
+      JSONParser p = getParser(in,i,-1);
+      if (p == null) break;
+      parse(p,in,expected);
+    }
+  }
+
+
+  public static void err(String input) throws IOException {
+    try {
+      JSONParser p = getParser(input);
+      while (p.nextEvent() != JSONParser.EOF) {}
+    } catch (Exception e) {
+      return;
+    }
+    fail("Input should failed:'" + input + "'");
+  }
+
+  @Test
+  public void testNull() throws IOException {
+    flags = JSONParser.FLAGS_STRICT;
+    err("nul");
+    err("n");
+    err("nullz");
+    err("[nullz]");
+    flags = JSONParser.FLAGS_DEFAULT;
+
+    parse("[null]","[0]");
+    parse("{'hi':null}",new Object[]{m,"hi",N,M,e});
+  }
+
+  @Test
+  public void testBool() throws IOException {
+    flags = JSONParser.FLAGS_STRICT;
+    err("[True]");
+    err("[False]");
+    err("[TRUE]");
+    err("[FALSE]");
+    err("[truex]");
+    err("[falsex]");
+    err("[tru]");
+    err("[fals]");
+    err("[tru");
+    err("[fals");
+    err("t");
+    err("f");
+    flags = JSONParser.FLAGS_DEFAULT;
+
+    parse("[false,true, false , true ]",new Object[]{a,f,t,f,t,A,e});
+  }
+
+  @Test
+  public void testString() throws IOException {
+    // NOTE: single quotes are converted to double quotes by this
+    // testsuite!
+    err("[']");
+    err("[',]");
+    err("{'}");
+    err("{',}");
+
+    err("['\\u111']");
+    err("['\\u11']");
+    err("['\\u1']");
+    err("['\\']");
+
+
+    flags = JSONParser.FLAGS_STRICT;
+    err("['\\ ']");  // escape of non-special char
+    err("['\\U1111']");  // escape of non-special char
+    flags = JSONParser.FLAGS_DEFAULT;
+
+    parse("['\\ ']", new Object[]{a, " ", A, e});  // escape of non-special char
+    parse("['\\U1111']", new Object[]{a, "U1111", A, e});  // escape of non-special char
+
+
+
+    parse("['']",new Object[]{a,"",A,e});
+    parse("['\\\\']",new Object[]{a,"\\",A,e});
+    parse("['X\\\\']",new Object[]{a,"X\\",A,e});
+    parse("['\\\\X']",new Object[]{a,"\\X",A,e});
+    parse("[\"\\\"\"]",new Object[]{a,"\"",A,e});
+
+    parse("['\\'']", true, new Object[]{a,"\"",A,e});
+    parse("['\\'']", false, new Object[]{a,"'",A,e});
+
+
+    String esc="\\n\\r\\tX\\b\\f\\/\\\\X\\\"";
+    String exp="\n\r\tX\b\f/\\X\"";
+    parse("['" + esc + "']",new Object[]{a,exp,A,e});
+    parse("['" + esc+esc+esc+esc+esc + "']",new Object[]{a,exp+exp+exp+exp+exp,A,e});
+
+    esc="\\u004A";
+    exp="\u004A";
+    parse("['" + esc + "']",new Object[]{a,exp,A,e});
+
+    esc="\\u0000\\u1111\\u2222\\u12AF\\u12BC\\u19DE";
+    exp="\u0000\u1111\u2222\u12AF\u12BC\u19DE";
+    parse("['" + esc + "']",new Object[]{a,exp,A,e});
+
+  }
+
+  @Test
+  public void testNumbers() throws IOException {
+    flags = JSONParser.FLAGS_STRICT;
+
+    err("[00]");
+    err("[003]");
+    err("[00.3]");
+    err("[1e1.1]");
+    err("[+1]");
+    err("[NaN]");
+    err("[Infinity]");
+    err("[--1]");
+
+    flags = JSONParser.FLAGS_DEFAULT;
+
+    String lmin    = "-9223372036854775808";
+    String lminNot = "-9223372036854775809";
+    String lmax    = "9223372036854775807";
+    String lmaxNot = "9223372036854775808";
+
+    String bignum="12345678987654321357975312468642099775533112244668800152637485960987654321";
+
+    parse("[0,1,-1,543,-876]", new Object[]{a,o(0),o(1),o(-1),o(543),o(-876),A,e});
+    parse("[-0]",new Object[]{a,o(0),A,e});
+
+
+    parse("["+lmin +"," + lmax+"]",
+        new Object[]{a,o(Long.MIN_VALUE),o(Long.MAX_VALUE),A,e});
+
+    parse("["+bignum+"]", new Object[]{a,bn(bignum),A,e});
+    parse("["+"-"+bignum+"]", new Object[]{a,bn("-"+bignum),A,e});
+
+    parse("["+lminNot+"]",new Object[]{a,bn(lminNot),A,e});
+    parse("["+lmaxNot+"]",new Object[]{a,bn(lmaxNot),A,e});
+
+    parse("["+lminNot + "," + lmaxNot + "]",
+        new Object[]{a,bn(lminNot),bn(lmaxNot),A,e});
+
+    // bignum many digits on either side of decimal
+    String t = bignum + "." + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    err("[" + t+".1" + "]"); // extra decimal
+    err("[" + "-"+t+".1" + "]");
+
+    // bignum exponent w/o fraction
+    t = "1" + "e+" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    t = "1" + "E+" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    t = "1" + "e" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    t = "1" + "E" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    t = "1" + "e-" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    t = "1" + "E-" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+    t = bignum + "e+" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    t = bignum + "E-" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    t = bignum + "e" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+    t = bignum + "." + bignum + "e" + bignum;
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+    err("[1E]");
+    err("[1E-]");
+    err("[1E+]");
+    err("[1E+.3]");
+    err("[1E+0.3]");
+    err("[1E+1e+3]");
+    err("["+bignum+"e"+"]");
+    err("["+bignum+"e-"+"]");
+    err("["+bignum+"e+"+"]");
+    err("["+bignum+"."+bignum+"."+bignum+"]");
+
+
+    double[] vals = new double[] {0,0.1,1.1,
+        Double.MAX_VALUE,
+        Double.MIN_VALUE,
+        2.2250738585072014E-308, /* Double.MIN_NORMAL */
+    };
+    for (int i=0; i<vals.length; i++) {
+      double d = vals[i];
+      parse("["+d+","+-d+"]", new Object[]{a,o(d),o(-d),A,e});
+    }
+
+    // MIN_NORMAL has the max number of digits (23), so check that
+    // adding an extra digit causes BIGNUM to be returned.
+    t = "2.2250738585072014E-308" + "0";
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+    // check it works with a leading zero too
+    t = "0.2250738585072014E-308" + "0";
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+    // check that overflow detection is working properly w/ numbers that don't cause a wrap to negatives
+    // when multiplied by 10
+    t = "1910151821265210155" + "0";
+    parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+    for (int i=0; i<1000000; i++) {
+      long val = random().nextLong();
+      String sval = Long.toString(val);
+      JSONParser parser = getParser("["+val+"]");
+      parser.nextEvent();
+      assertTrue(parser.nextEvent() == JSONParser.LONG);
+      if (random().nextBoolean()) {
+        assertEquals(val, parser.getLong());
+      } else {
+        CharArr chars = parser.getNumberChars();
+        assertEquals(sval, chars.toString());
+      }
+    }
+
+  }
+
+  @Test
+  public void testArray() throws IOException {
+    parse("[]","[]");
+    parse("[ ]","[]");
+    parse(" \r\n\t[\r\t\n ]\r\n\t ","[]");
+
+    parse("[0]","[l]");
+    parse("['0']","[s]");
+    parse("[0,'0',0.1]","[lsn]");
+
+    parse("[[[[[]]]]]","[[[[[]]]]]");
+    parse("[[[[[0]]]]]","[[[[[l]]]]]");
+
+    err("]");
+    err("[");
+    err("[[]");
+    err("[]]");
+    err("[}");
+    err("{]");
+    err("['a':'b']");
+
+    flags=JSONParser.FLAGS_STRICT;
+    err("[,]");         // test that extra commas fail
+    err("[[],]");
+    err("['a',]");
+    err("['a',]");
+    flags=JSONParser.FLAGS_DEFAULT;
+
+    parse("[,]","[]");  // test extra commas
+    parse("[,,]","[]");
+    parse("[,,,]","[]");
+    parse("[[],]","[[]]");
+    parse("[[,],]","[[]]");
+    parse("[[,,],,]","[[]]");
+    parse("[,[,,],,]","[[]]");
+    parse("[,5,[,,5],,]","[l[l]]");
+
+  }
+
+  @Test
+  public void testObject() throws IOException {
+    parse("{}","{}");
+    parse("{}","{}");
+    parse(" \r\n\t{\r\t\n }\r\n\t ","{}");
+
+    parse("{'':null}","{s0}");
+
+    err("}");
+    err("[}]");
+    err("{");
+    err("[{]");
+    err("{{}");
+    err("[{{}]");
+    err("{}}");
+    err("[{}}]");
+    err("{1}");
+    err("[{1}]");
+    err("{'a'}");
+    err("{'a','b'}");
+    err("{[]:'b'}");
+    err("{{'a':'b'}:'c'}");
+    err("{'a','b'}}");
+
+    // bare strings allow these to pass
+    flags=JSONParser.FLAGS_STRICT;
+    err("{null:'b'}");
+    err("{true:'b'}");
+    err("{false:'b'}");
+    err("{,}");         // test that extra commas fail
+    err("{{},}");
+    err("{'a':'b',}");
+    flags=JSONParser.FLAGS_DEFAULT;
+
+    parse("{}", new Object[]{m,M,e});
+    parse("{,}", new Object[]{m,M,e});
+    parse("{,,}", new Object[]{m,M,e});
+    parse("{'a':{},}", new Object[]{m,"a",m,M,M,e});
+    parse("{'a':{},,}", new Object[]{m,"a",m,M,M,e});
+    parse("{,'a':{,},,}", new Object[]{m,"a",m,M,M,e});
+    parse("{'a':'b'}", new Object[]{m,"a","b",M,e});
+    parse("{'a':5}", new Object[]{m,"a",o(5),M,e});
+    parse("{'a':null}", new Object[]{m,"a",N,M,e});
+    parse("{'a':[]}", new Object[]{m,"a",a,A,M,e});
+    parse("{'a':{'b':'c'}}", new Object[]{m,"a",m,"b","c",M,M,e});
+
+    String big = "Now is the time for all good men to come to the aid of their country!";
+    String bigger = big+big+big+big+big;
+    parse("{'"+bigger+"':'"+bigger+"','a':'b'}", new Object[]{m,bigger,bigger,"a","b",M,e});
+
+
+    flags=JSONParser.ALLOW_UNQUOTED_KEYS;
+    parse("{a:'b'}", new Object[]{m,"a","b",M,e});
+    parse("{null:'b'}", new Object[]{m,"null","b",M,e});
+    parse("{true: 'b'}", new Object[]{m,"true","b",M,e});
+    parse("{ false :'b'}", new Object[]{m,"false","b",M,e});
+    parse("{null:null, true : true , false : false , x:'y',a:'b'}", new Object[]{m,"null",N,"true",t,"false",f,"x","y","a","b",M,e});
+    flags = JSONParser.FLAGS_DEFAULT | JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT;
+    parse("{'a'{'b':'c'}}", new Object[]{m, "a", m, "b", "c", M, M, e});
+    parse("{'a': [{'b':'c'} {'b':'c'}]}", new Object[]{m, "a",a, m, "b", "c", M, m, "b", "c", M,A, M, e});
+    parse("{'a' [{'b':'c'} {'b':'c'}]}", new Object[]{m, "a", a, m, "b", "c", M, m, "b", "c", M, A, M, e});
+    parse("{'a':[['b']['c']]}", new Object[]{m, "a", a, a, "b", A, a, "c", A, A, M, e});
+    parse("{'a': {'b':'c'} 'd': {'e':'f'}}", new Object[]{m, "a", m, "b", "c",M,  "d", m,"e","f", M, M, e});
+    parse("{'a': {'b':'c'} d: {'e':'f'}}", new Object[]{m, "a", m, "b", "c",M,  "d", m,"e","f", M, M, e});
+
+    flags = JSONParser.FLAGS_DEFAULT | JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT | JSONParser.OPTIONAL_OUTER_BRACES;
+    parse("'a':{'b':'c'}", new Object[]{m, "a", m, "b", "c", M, M, e});
+    parse("'a':{'b':'c'}", true, new Object[]{m, "a", m, "b", "c", M, M, e});
+    parse("a:'b'", new Object[]{m, "a", "b", M, e});
+
+
+    flags = JSONParser.FLAGS_DEFAULT;
+
+  }
+
+  @Test
+  public void testBareString() throws Exception {
+    flags=JSONParser.ALLOW_UNQUOTED_STRING_VALUES | JSONParser.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER;
+    String[] strings = new String[] {"t","f","n","a","tru","fals","nul","abc","trueX","falseXY","nullZ","truetrue", "$true", "a.bc.def","a_b-c/d"};
+
+    for (String s : strings) {
+      parse(s, new Object[]{s, e});
+      parse("[" + s + "]", new Object[]{a, s, A, e});
+      parse("[ " + s + ", "+s +" ]", new Object[]{a, s, s, A, e});
+      parse("[" + s + ","+s +"]",    new Object[]{a, s, s, A, e});
+      parse("\u00a0[\u00a0\r\n\t\u00a0" + s + "\u00a0,\u00a0\u00a0"+s +"\u00a0]\u00a0",    new Object[]{a, s, s, A, e});
+    }
+
+    flags |= JSONParser.ALLOW_UNQUOTED_KEYS;
+    for (String s : strings) {
+      parse("{" + s + ":" + s + "}", new Object[]{m, s, s, M, e});
+      parse("{ " + s + " \t\n\r:\t\n\r " + s + "\t\n\r}", new Object[]{m, s, s, M, e});
+    }
+
+    parse("{true:true, false:false, null:null}",new Object[]{m,"true",t,"false",f,"null",N,M,e});
+
+    flags=JSONParser.FLAGS_DEFAULT;
+  }
+
+
+  @Test
+  public void testAPI() throws IOException {
+    JSONParser p = new JSONParser("[1,2]");
+    assertEquals(JSONParser.ARRAY_START, p.nextEvent());
+    // no nextEvent for the next objects...
+    assertEquals(1,p.getLong());
+    assertEquals(2,p.getLong());
+  }
+
+
+  @Test
+  public void testComments() throws IOException {
+    parse("#pre comment\n{//before a\n  'a' /* after a **/ #before separator\n  : /* after separator */ {/*before b*/'b'#after b\n://before c\n'c'/*after c*/}/*after close*/}#post comment no EOL", new Object[]{m,"a",m,"b","c",M,M,e});
+  }
+
+  // rfc7159 permits any JSON values to be top level values
+  @Test
+  public void testTopLevelValues() throws Exception {
+    parse("\"\"", new Object[]{""});
+    parse("\"hello\"", new Object[]{"hello"});
+    parse("true", new Object[]{t});
+    parse("false", new Object[]{f});
+    parse("null", new Object[]{N});
+    parse("42", new Object[]{42L});
+    parse("1.414", new Object[]{1.414d});
+    parse("/*comment*/1.414/*more comment*/", new Object[]{1.414d});
+  }
+}

solr/solrj/src/test/org/noggit/TestJSONWriter.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+public class TestJSONWriter extends SolrTestCaseJ4 {
+
+  // note - TestObjectBuilder also exercises JSONWriter
+
+  public void test(String expected, Object val, int indent) throws IOException {
+    expected = expected.replace('\'','"');
+    String s1 = JSONUtil.toJSON(val, indent);
+    assertEquals(s1, expected);
+  }
+
+  public static List<Object> L(Object... lst) {
+    return Arrays.asList(lst);
+  }
+  public static Object[] A(Object... lst) {
+    return lst;
+  }
+  public static Map<String,Object> O(Object... lst) {
+    LinkedHashMap<String,Object> map = new LinkedHashMap<String,Object>();
+    for (int i=0; i<lst.length; i+=2) {
+      map.put(lst[i].toString(), lst[i+1]);
+    }
+    return map;
+  }
+
+  // NOTE: the specifics of indentation may change in the future!
+  @Test
+  public void testWriter() throws Exception {
+    test("[]",L(),2);
+    test("{}",O(),2);
+    test("[\n  10,\n  20]", L(10,20), 2);
+    test("{\n 'a':10,\n 'b':{\n  'c':20,\n  'd':30}}", O("a",10,"b",O("c",20,"d",30)), 1);
+
+    test("['\\r\\n\\u0000\\'']", L("\r\n\u0000\""),2);
+
+  }
+
+
+  public static class Unknown {
+    @Override
+    public String toString() {
+      return "a,\"b\",c";
+    }
+  }
+
+  public static class Custom implements JSONWriter.Writable {
+    @Override
+    public void write(JSONWriter writer) {
+      Map<String,Integer> val = new LinkedHashMap<String,Integer>();
+      val.put("a",1);
+      val.put("b",2);
+      writer.write(val);
+    }
+  }
+
+  @Test
+  public void testWritable() throws Exception {
+    test("[{'a':1,'b':2}]", L(new Custom()), -1);
+    test("[10,{'a':1,'b':2},20]", L(10, new Custom(), 20), -1);
+  }
+
+  @Test
+  public void testUnknown() throws Exception {
+    test("['a,\\\"b\\\",c']", L(new Unknown()), -1);
+  }
+
+}

solr/solrj/src/test/org/noggit/TestObjectBuilder.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+public class TestObjectBuilder extends SolrTestCaseJ4 {
+
+  public void _test(String val, Object expected) throws IOException {
+    val = val.replace('\'','"');
+    Object v = ObjectBuilder.fromJSON(val);
+
+    String s1 = JSONUtil.toJSON(v,-1);
+    String s2 = JSONUtil.toJSON(expected,-1);
+    assertEquals(s1, s2);
+
+    // not make sure that it round-trips correctly
+    JSONParser p2 = TestJSONParser.getParser(s1);
+    Object v2 = ObjectBuilder.getVal(p2);
+    String s3 = JSONUtil.toJSON(v2,-1);
+    assertEquals(s1, s3);
+  }
+
+  public static List<Object> L(Object... lst) {
+    return Arrays.asList(lst);
+  }
+  public static Object[] A(Object... lst) {
+    return lst;
+  }
+  public static Map<String, Object> O(Object... lst) {
+    LinkedHashMap<String,Object> map = new LinkedHashMap<String,Object>();
+    for (int i=0; i<lst.length; i+=2) {
+      map.put(lst[i].toString(), lst[i+1]);
+    }
+    return map;
+  }
+  @Test
+  public void testWithoutStartingBraces() throws IOException {
+    JSONParser parser = new JSONParser("a: {key:val1}b:{key:val2}");
+    parser.setFlags(JSONParser.FLAGS_DEFAULT | JSONParser.OPTIONAL_OUTER_BRACES| JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT);
+    ObjectBuilder objectBuilder = new ObjectBuilder(parser);
+    String s1 = JSONUtil.toJSON(objectBuilder.getObject(),-1);
+    String expected = JSONUtil.toJSON(O("a", O("key", "val1"), "b", O("key", "val2")),-1);
+    assertEquals(s1, expected);
+  }
+
+  public void _testVariations(String str, Object expected) throws IOException {
+    _test("["+str+"]", L(expected));
+    _test("["+str+","+str+"]", L(expected, expected));
+    _test("["+str+","+str+"]", A(expected, expected));
+    _test("{'foo':"+str+"}", O("foo",expected));
+    _test("{'foo':"+str+",'bar':{'a':"+str+"},'baz':["+str+"],'zzz':["+str+"]}",
+        O("foo",expected,"bar",O("a",expected),"baz", L(expected), "zzz", A(expected)));
+
+  }
+  @Test
+  public void testBuilder() throws IOException {
+    _testVariations("[]", L());
+    _testVariations("[]", L());
+    _testVariations("{}", O());
+    _testVariations("[[]]", L(L()));
+    _testVariations("{'foo':{}}", O("foo",O()));
+    _testVariations("[false,true,1,1.4,null,'hi']", L(false, true, 1, 1.4, null, "hi"));
+    _testVariations("'hello'", "hello".toCharArray());
+
+    // test array types
+    _testVariations("[[10,20],['a','b']]", L(A(10,20),A("a","b")));
+    _testVariations("[1,2,3]", new int[]{1,2,3});
+    _testVariations("[1,2,3]", new long[]{1,2,3});
+    _testVariations("[1.0,2.0,3.0]", new float[]{1,2,3});
+    _testVariations("[1.0,2.0,3.0]", new double[]{1,2,3});
+    _testVariations("[1,2,3]", new short[]{1,2,3});
+    _testVariations("[1,2,3]", new byte[]{1,2,3});
+    _testVariations("[false,true,false]", new boolean[]{false,true,false});
+  }
+
+}