From a73f5e279c510a8746bbf0e4754d205248a0d355 Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Sun, 14 Nov 2010 20:45:03 +0000 Subject: [PATCH] SOLR-2236: random testing for grouping git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1035064 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/solr/TestGroupingSearch.java | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/solr/src/test/org/apache/solr/TestGroupingSearch.java b/solr/src/test/org/apache/solr/TestGroupingSearch.java index d4af548f4f0..fd634c28a5e 100644 --- a/solr/src/test/org/apache/solr/TestGroupingSearch.java +++ b/solr/src/test/org/apache/solr/TestGroupingSearch.java @@ -18,10 +18,17 @@ package org.apache.solr; import org.apache.lucene.search.FieldCache; +import org.apache.noggit.JSONUtil; +import org.apache.noggit.ObjectBuilder; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import java.util.*; + public class TestGroupingSearch extends SolrTestCaseJ4 { @BeforeClass @@ -301,4 +308,184 @@ public class TestGroupingSearch extends SolrTestCaseJ4 { + @Test + public void testRandomGrouping() throws Exception { + /** + updateJ("{\"add\":{\"doc\":{\"id\":\"77\"}}}", params("commit","true")); + assertJQ(req("q","id:77"), "/response/numFound==1"); + + Doc doc = createDocObj(types); + updateJ(toJSON(doc), params("commit","true")); + + assertJQ(req("q","id:"+doc.id), "/response/numFound==1"); + **/ + + int indexIter=0; // make >0 to enable test + int queryIter=1000; + + while (--indexIter >= 0) { + + List types = new ArrayList(); + types.add(new FldType("id",ONE_ONE, new SVal('A','Z',2,2))); + types.add(new FldType("score_f",ONE_ONE, new FVal(1,100))); // field used to score + types.add(new FldType("foo_i",ONE_ONE, new IRange(0,10))); + types.add(new FldType("foo_s",ONE_ONE, new SVal('a','z',1,2))); + + Map model = indexDocs(types, null, 2); + System.out.println("############### model=" + model); + + + for (int qiter=0; qiter groups = groupBy(model.values(), groupField); + int rows = random.nextInt(11)-1; + int start = random.nextInt(5)==0 ? random.nextInt(model.size()+2) : random.nextInt(5); // pick a small start normally for better coverage + int group_limit = random.nextInt(11)-1; +group_limit = random.nextInt(10)+1; + int group_offset = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(2); // pick a small start normally for better coverage + + // sort each group + String[] stringSortA = new String[1]; + Comparator groupComparator = createSort(h.getCore().getSchema(), types, stringSortA); + String groupSortStr = stringSortA[0]; + + // Test specific sort + /*** + groupComparator = createComparator("_docid_", false, false, false); + stringSort = "_docid_ desc"; + ***/ + + // first sort the docs in each group + for (Grp grp : groups.values()) { + Collections.sort(grp.docs, groupComparator); + } + + // now sort the groups by the first doc in that group + Comparator sortComparator = random.nextBoolean() ? groupComparator : createSort(h.getCore().getSchema(), types, stringSortA); + String sortStr = stringSortA[0]; + + List sortedGroups = new ArrayList(groups.values()); + Collections.sort(sortedGroups, createFirstDocComparator(sortComparator)); + + Object modelResponse = buildGroupedResult(h.getCore().getSchema(), sortedGroups, start, rows, group_offset, group_limit); + + // TODO: create a random filter too + + SolrQueryRequest req = req("group","true","wt","json","indent","true", "q","{!func}score_f", "group.field",groupField + ,sortStr==null ? "nosort":"sort", sortStr ==null ? "": sortStr + ,(groupSortStr==null || groupSortStr==sortStr) ? "nosort":"group.sort", groupSortStr==null ? "": groupSortStr + ,"rows",""+rows, "start",""+start, "group.offset",""+group_offset, "group.limit",""+group_limit + ); + + String strResponse = h.query(req); + + Object realResponse = ObjectBuilder.fromJSON(strResponse); + String err = JSONTestUtil.matchObj("/grouped/"+groupField, realResponse, modelResponse); + if (err != null) { + log.error("GROUPING MISMATCH: " + err + + "\n\tresult="+strResponse + + "\n\texpected="+ JSONUtil.toJSON(modelResponse) + + "\n\tsorted_model="+ sortedGroups + ); + + fail(err); + } + } // end query iter + } // end index iter + + } + + public static Object buildGroupedResult(IndexSchema schema, List sortedGroups, int start, int rows, int group_offset, int group_limit) { + Map result = new LinkedHashMap(); + + long matches = 0; + for (Grp grp : sortedGroups) { + matches += grp.docs.size(); + } + result.put("matches", matches); + List groupList = new ArrayList(); + result.put("groups", groupList); + + for (int i=start; i= rows) break; // directly test rather than calculating, so we can catch any calc errors in the real code + Map group = new LinkedHashMap(); + groupList.add(group); + + Grp grp = sortedGroups.get(i); + group.put("groupValue", grp.groupValue); + + Map resultSet = new LinkedHashMap(); + group.put("doclist", resultSet); + resultSet.put("numFound", grp.docs.size()); + resultSet.put("start", start); + + List docs = new ArrayList(); + resultSet.put("docs", docs); + for (int j=group_offset; j= group_limit) break; + docs.add( grp.docs.get(j).toObject(schema) ); + } + } + + return result; + } + + + public static Comparator createFirstDocComparator(final Comparator docComparator) { + return new Comparator() { + @Override + public int compare(Grp o1, Grp o2) { + // all groups should have at least one doc + Doc d1 = o1.docs.get(0); + Doc d2 = o2.docs.get(0); + return docComparator.compare(d1, d2); + } + }; + } + + + + public static Map groupBy(Collection docs, String field) { + Map groups = new HashMap(); + for (Doc doc : docs) { + List vals = doc.getValues(field); + if (vals == null) { + Grp grp = groups.get(null); + if (grp == null) { + grp = new Grp(); + grp.groupValue = null; + grp.docs = new ArrayList(); + groups.put(null, grp); + } + grp.docs.add(doc); + } else { + for (Comparable val : vals) { + + Grp grp = groups.get(val); + if (grp == null) { + grp = new Grp(); + grp.groupValue = val; + grp.docs = new ArrayList(); + groups.put(grp.groupValue, grp); + } + grp.docs.add(doc); + } + } + } + return groups; + } + + + public static class Grp { + public Comparable groupValue; + public List docs; + + @Override + public String toString() { + return "{groupValue="+groupValue+",docs="+docs+"}"; + } + } } + +