diff --git a/solr/src/test/org/apache/solr/JSONTestUtil.java b/solr/src/test/org/apache/solr/JSONTestUtil.java index 2eef7d6380a..65b9d398714 100644 --- a/solr/src/test/org/apache/solr/JSONTestUtil.java +++ b/solr/src/test/org/apache/solr/JSONTestUtil.java @@ -121,9 +121,13 @@ class CollectionTester { } boolean match() { - if (expected == null && val == null) { + if (expected == val) { return true; } + if (expected == null || val == null) { + setErr("mismatch: '" + expected + "'!='" + val + "'"); + return false; + } if (expected instanceof List) { return matchList(); } @@ -133,8 +137,20 @@ class CollectionTester { // generic fallback if (!expected.equals(val)) { - setErr("mismatch: '" + expected + "'!='" + val + "'"); - return false; + + // make an exception for some numerics + if (expected instanceof Integer && val instanceof Long || expected instanceof Long && val instanceof Integer + && ((Number)expected).longValue() == ((Number)val).longValue()) + { + // OK + } else if (expected instanceof Float && val instanceof Double || expected instanceof Double && val instanceof Float + && ((Number)expected).doubleValue() == ((Number)val).doubleValue()) + { + // OK + } else { + setErr("mismatch: '" + expected + "'!='" + val + "'"); + return false; + } } // setErr("unknown expected type " + expected.getClass().getName()); diff --git a/solr/src/test/org/apache/solr/SolrTestCaseJ4.java b/solr/src/test/org/apache/solr/SolrTestCaseJ4.java index c46617901c1..674bc71dadc 100755 --- a/solr/src/test/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/src/test/org/apache/solr/SolrTestCaseJ4.java @@ -20,6 +20,10 @@ package org.apache.solr; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.apache.noggit.CharArr; +import org.apache.noggit.JSONUtil; +import org.apache.noggit.JSONWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; @@ -27,9 +31,20 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.XML; import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.JsonUpdateRequestHandler; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.request.BinaryQueryResponseWriter; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.servlet.DirectSolrConnection; +import org.apache.solr.servlet.SolrRequestParsers; import org.apache.solr.util.TestHarness; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -40,10 +55,9 @@ import org.xml.sax.SAXException; import javax.xml.xpath.XPathExpressionException; import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.io.StringWriter; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; +import java.util.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -476,7 +490,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { * @see #doc */ public static String adoc(String... fieldsAndValues) { - Doc d = doc(fieldsAndValues); + XmlDoc d = doc(fieldsAndValues); return add(d); } @@ -504,7 +518,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { * @see #add * @see #doc */ - public static String add(Doc doc, String... args) { + public static String add(XmlDoc doc, String... args) { try { StringWriter r = new StringWriter(); @@ -547,8 +561,8 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { * @param fieldsAndValues 0th and Even numbered args are fields names, Odds are field values. * @see TestHarness#makeSimpleDoc */ - public static Doc doc(String... fieldsAndValues) { - Doc d = new Doc(); + public static XmlDoc doc(String... fieldsAndValues) { + XmlDoc d = new XmlDoc(); d.xml = h.makeSimpleDoc(fieldsAndValues).toString(); return d; } @@ -597,7 +611,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { } /** Neccessary to make method signatures un-ambiguous */ - public static class Doc { + public static class XmlDoc { public String xml; public String toString() { return xml; } } @@ -617,4 +631,394 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { public void clearIndex() { assertU(delQ("*:*")); } + + /** Send JSON update commands */ + public static String updateJ(String json, SolrParams args) throws Exception { + SolrCore core = h.getCore(); + DirectSolrConnection connection = new DirectSolrConnection(core); + SolrRequestHandler handler = core.getRequestHandler("/udate/json"); + if (handler == null) { + handler = new JsonUpdateRequestHandler(); + handler.init(null); + } + return connection.request(handler, args, json); + } + + + ///////////////////////////////////////////////////////////////////////////////////// + //////////////////////////// random document / index creation /////////////////////// + ///////////////////////////////////////////////////////////////////////////////////// + + public abstract static class Vals { + public abstract Comparable get(); + public String toJSON(Comparable val) { + return JSONUtil.toJSON(val); + } + + protected int between(int min, int max) { + return min != max ? random.nextInt(max-min+1) + min : min; + } + } + + public abstract static class IVals extends Vals { + public abstract int getInt(); + } + + public static class IRange extends IVals { + final int min; + final int max; + public IRange(int min, int max) { + this.min = min; + this.max = max; + } + + @Override + public int getInt() { + return between(min,max); + } + + @Override + public Comparable get() { + return getInt(); + } + } + + public static class FVal extends Vals { + final float min; + final float max; + public FVal(float min, float max) { + this.min = min; + this.max = max; + } + + public float getFloat() { + if (min >= max) return min; + return min + random.nextFloat() * (max - min); + } + + @Override + public Comparable get() { + return getFloat(); + } + } + + public static class SVal extends Vals { + char start; + char end; + int minLength; + int maxLength; + + public SVal() { + this('a','z',1,10); + } + + public SVal(char start, char end, int minLength, int maxLength) { + this.start = start; + this.end = end; + this.minLength = minLength; + this.maxLength = maxLength; + } + + @Override + public Comparable get() { + char[] arr = new char[between(minLength,maxLength)]; + for (int i=0; i fields; + public int order; // the order this document was added to the index + + + public String toString() { + return "Doc("+order+"):"+fields.toString(); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Doc)) return false; + Doc other = (Doc)o; + return this==other || id != null && id.equals(other.id); + } + + @Override + public int compareTo(Object o) { + if (!(o instanceof Doc)) return this.getClass().hashCode() - o.getClass().hashCode(); + Doc other = (Doc)o; + return this.id.compareTo(other.id); + } + + public List getValues(String field) { + for (Fld fld : fields) { + if (fld.ftype.fname.equals(field)) return fld.vals; + } + return null; + } + + public Comparable getFirstValue(String field) { + List vals = getValues(field); + return vals==null || vals.size()==0 ? null : vals.get(0); + } + + public Map toObject(IndexSchema schema) { + Map result = new HashMap(); + for (Fld fld : fields) { + SchemaField sf = schema.getField(fld.ftype.fname); + if (!sf.multiValued()) { + result.put(fld.ftype.fname, fld.vals.get(0)); + } else { + result.put(fld.ftype.fname, fld.vals); + } + } + return result; + } + + } + + public static class Fld { + public FldType ftype; + public List vals; + public String toString() { + return ftype.fname + "=" + (vals.size()==1 ? vals.get(0).toString() : vals.toString()); + } + } + + class FldType { + public String fname; + public IRange numValues; + public Vals vals; + + public FldType(String fname, Vals vals) { + this(fname, ZERO_ONE, vals); + } + + public FldType(String fname, IRange numValues, Vals vals) { + this.fname = fname; + this.numValues = numValues; + this.vals = vals; + } + + public Comparable createValue() { + return vals.get(); + } + + public List createValues() { + int nVals = numValues.getInt(); + if (nVals <= 0) return null; + List vals = new ArrayList(nVals); + for (int i=0; i vals = createValues(); + if (vals == null) return null; + + Fld fld = new Fld(); + fld.ftype = this; + fld.vals = vals; + return fld; + } + + } + + + public Map indexDocs(List descriptor, Map model, int nDocs) throws Exception { + if (model == null) { + model = new LinkedHashMap(); + } + + // commit an average of 10 times for large sets, or 10% of the time for small sets + int commitOneOutOf = Math.max(nDocs/10, 10); + + + // find the max order (docid) and start from there + int order = -1; + for (Doc doc : model.values()) { + order = Math.max(order, doc.order); + } + order++; + + for (int i=0; i descriptor) { + Doc doc = new Doc(); + doc.fields = new ArrayList(); + for (FldType ftype : descriptor) { + Fld fld = ftype.createField(); + if (fld != null) { + doc.fields.add(fld); + if ("id".equals(ftype.fname)) + doc.id = fld.vals.get(0); + } + } + return doc; + } + + public static Comparator createSort(IndexSchema schema, List fieldTypes, String[] out) { + StringBuilder sortSpec = new StringBuilder(); + int nSorts = random.nextInt(4); + List> comparators = new ArrayList>(); + for (int i=0; i0) sortSpec.append(','); + + int which = random.nextInt(fieldTypes.size()+2); + boolean asc = random.nextBoolean(); + if (which == fieldTypes.size()) { + // sort by score + sortSpec.append("score").append(asc ? " asc" : " desc"); + comparators.add(createComparator("score", asc, false, false)); + } else if (which == fieldTypes.size() + 1) { + // sort by docid + sortSpec.append("_docid_").append(asc ? " asc" : " desc"); + comparators.add(createComparator("_docid_", asc, false, false)); + } else { + String field = fieldTypes.get(which).fname; + sortSpec.append(field).append(asc ? " asc" : " desc"); + SchemaField sf = schema.getField(field); + comparators.add(createComparator(field, asc, sf.sortMissingLast(), sf.sortMissingFirst())); + } + } + + out[0] = sortSpec.length() > 0 ? sortSpec.toString() : null; + + if (comparators.size() == 0) { + // default sort is by score desc + comparators.add(createComparator("score", false, false, false)); + } + + return createComparator(comparators); + } + + public static Comparator createComparator(final String field, final boolean asc, final boolean sortMissingLast, final boolean sortMissingFirst) { + final int mul = asc ? 1 : -1; + + if (field.equals("_docid_")) { + return new Comparator() { + @Override + public int compare(Doc o1, Doc o2) { + return (o1.order - o2.order) * mul; + } + }; + } + + if (field.equals("score")) { + return createComparator("score_f", asc, sortMissingLast, sortMissingFirst); + } + + return new Comparator() { + @Override + public int compare(Doc o1, Doc o2) { + Comparable v1 = o1.getFirstValue(field); + Comparable v2 = o2.getFirstValue(field); + + int c = 0; + if (v1 == v2) { + c = 0; + } else if (v1 == null) { + if (sortMissingLast) c = mul; + else if (sortMissingFirst) c = -mul; + else c = -1; + } else if (v2 == null) { + if (sortMissingLast) c = -mul; + else if (sortMissingFirst) c = mul; + else c = 1; + } else { + c = v1.compareTo(v2); + } + + c = c * mul; + + return c; + } + }; + } + + public static Comparator createComparator(final List> comparators) { + return new Comparator() { + @Override + public int compare(Doc o1, Doc o2) { + int c = 0; + for (Comparator comparator : comparators) { + c = comparator.compare(o1, o2); + if (c!=0) return c; + } + return o1.order - o2.order; + } + }; + } + + + public static String toJSON(Doc doc) { + CharArr out = new CharArr(); + try { + out.append("{\"add\":{\"doc\":{"); + boolean firstField = true; + for (Fld fld : doc.fields) { + if (firstField) firstField=false; + else out.append(','); + JSONUtil.writeString(fld.ftype.fname, 0, fld.ftype.fname.length(), out); + out.append(':'); + if (fld.vals.size() > 1) { + out.append('['); + } + boolean firstVal = true; + for (Comparable val : fld.vals) { + if (firstVal) firstVal=false; + else out.append(','); + out.append(JSONUtil.toJSON(val)); + } + if (fld.vals.size() > 1) { + out.append(']'); + } + } + out.append("}}}"); + } catch (IOException e) { + // should never happen + } + return out.toString(); + } + + + } diff --git a/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java b/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java index b8795aea7ea..39b1cd93300 100644 --- a/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java +++ b/solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java @@ -143,11 +143,19 @@ public class DirectSolrConnection path= pathAndParams; params = new MapSolrParams( new HashMap() ); } - + + return request(path, params, body); + } + + + public String request(String path, SolrParams params, String body) throws Exception + { // Extract the handler from the path or params SolrRequestHandler handler = core.getRequestHandler( path ); if( handler == null ) { if( "/select".equals( path ) || "/select/".equalsIgnoreCase( path) ) { + if (params == null) + params = new MapSolrParams( new HashMap() ); String qt = params.get( CommonParams.QT ); handler = core.getRequestHandler( qt ); if( handler == null ) { @@ -158,13 +166,21 @@ public class DirectSolrConnection if( handler == null ) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "unknown handler: "+path ); } - + + return request(handler, params, body); + } + + public String request(SolrRequestHandler handler, SolrParams params, String body) throws Exception + { + if (params == null) + params = new MapSolrParams( new HashMap() ); + // Make a stream for the 'body' content List streams = new ArrayList( 1 ); if( body != null && body.length() > 0 ) { streams.add( new ContentStreamBase.StringStream( body ) ); } - + SolrQueryRequest req = null; try { req = parser.buildRequestFrom( core, params, streams ); @@ -173,7 +189,7 @@ public class DirectSolrConnection if( rsp.getException() != null ) { throw rsp.getException(); } - + // Now write it out QueryResponseWriter responseWriter = core.getQueryResponseWriter(req); StringWriter out = new StringWriter(); @@ -185,7 +201,9 @@ public class DirectSolrConnection } } } - + + + /** * Use this method to close the underlying SolrCore. *