From d593f195ba81f78a4facb906d295bb2aa8edf9a2 Mon Sep 17 00:00:00 2001 From: "Chris M. Hostetter" Date: Tue, 28 Jan 2014 19:03:35 +0000 Subject: [PATCH] SOLR-5652: test fixes: add 'plain' docvalue field variants to schema, and check codecs for support of missing docValues before trying to sort on them git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1562155 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/util/LuceneTestCase.java | 12 ++ .../solr/collection1/conf/schema-sorts.xml | 23 ++++ .../org/apache/solr/CursorPagingTest.java | 117 +++++++++++++++--- .../solr/cloud/DistribCursorPagingTest.java | 95 +++++++++++--- 4 files changed, 217 insertions(+), 30 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index b581b7a60af..b37d461b6b4 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -1365,6 +1365,18 @@ public abstract class LuceneTestCase extends Assert { } } + /** Returns true if the default codec supports single valued docvalues with missing values */ + public static boolean defaultCodecSupportsMissingDocValues() { + String name = Codec.getDefault().getName(); + if (name.equals("Lucene3x") || + name.equals("Lucene40") || name.equals("Appending") || + name.equals("Lucene41") || + name.equals("Lucene42")) { + return false; + } + return true; + } + /** Returns true if the default codec supports SORTED_SET docvalues */ public static boolean defaultCodecSupportsSortedSet() { String name = Codec.getDefault().getName(); diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml index 88ccc52a000..f5b711c3769 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml @@ -32,36 +32,42 @@ NOTE: Tests expect every field in this schema to be sortable. + + + + + + @@ -73,26 +79,37 @@ NOTE: Tests expect every field in this schema to be sortable. + + + + + + + + + + + @@ -101,36 +118,42 @@ NOTE: Tests expect every field in this schema to be sortable. + + + + + + diff --git a/solr/core/src/test/org/apache/solr/CursorPagingTest.java b/solr/core/src/test/org/apache/solr/CursorPagingTest.java index 5f728bbc6bc..f80d702777c 100644 --- a/solr/core/src/test/org/apache/solr/CursorPagingTest.java +++ b/solr/core/src/test/org/apache/solr/CursorPagingTest.java @@ -120,6 +120,9 @@ public class CursorPagingTest extends SolrTestCaseJ4 { String cursorMark; SolrParams params = null; + final String intsort = "int" + (random().nextBoolean() ? "" : "_dv"); + final String intmissingsort = defaultCodecSupportsMissingDocValues() ? intsort : "int"; + // trivial base case: ensure cursorMark against an empty index doesn't blow up cursorMark = CURSOR_MARK_START; params = params("q", "*:*", @@ -145,7 +148,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { assertU(adoc("id", "6", "str", "a", "float", "64.5", "int", "7")); assertU(adoc("id", "1", "str", "a", "float", "64.5", "int", "7")); assertU(adoc("id", "4", "str", "a", "float", "11.1", "int", "6")); - assertU(adoc("id", "3", "str", "a", "float", "11.1", "int", "3")); + assertU(adoc("id", "3", "str", "a", "float", "11.1")); // int is missing assertU(commit()); // base case: ensure cursorMark that matches no docs doesn't blow up @@ -241,7 +244,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { "facet", "true", "facet.field", "str", "json.nl", "map", - "sort", "int asc, id asc"); + "sort", intsort + " asc, id asc"); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==8" ,"/response/start==0" @@ -269,6 +272,66 @@ public class CursorPagingTest extends SolrTestCaseJ4 { ,"/facet_counts/facet_fields/str=={'a':4,'b':1,'c':3}" )); + // int missing first sort with dups, id tie breaker + cursorMark = CURSOR_MARK_START; + params = params("q", "-int:2001 -int:4055", + "rows","3", + "fl", "id", + "json.nl", "map", + "sort", intmissingsort + "_first asc, id asc"); + cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[{'id':3},{'id':7},{'id':0}]" + ); + cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[{'id':4},{'id':1},{'id':6}]" + ); + cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[{'id':9},{'id':2}]" + ); + // no more, so no change to cursorMark, and no new docs + assertEquals(cursorMark, + assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[]" + )); + + // int missing last sort with dups, id tie breaker + cursorMark = CURSOR_MARK_START; + params = params("q", "-int:2001 -int:4055", + "rows","3", + "fl", "id", + "json.nl", "map", + "sort", intmissingsort + "_last asc, id asc"); + cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[{'id':7},{'id':0},{'id':4}]" + ); + cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[{'id':1},{'id':6},{'id':9}]" + ); + cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[{'id':2},{'id':3}]" + ); + // no more, so no change to cursorMark, and no new docs + assertEquals(cursorMark, + assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) + ,"/response/numFound==8" + ,"/response/start==0" + ,"/response/docs==[]" + )); + // string sort with dups, id tie breaker cursorMark = CURSOR_MARK_START; params = params("q", "*:*", @@ -298,7 +361,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { params = params("q", "*:*", "rows","2", "fl", "id", - "sort", "float asc, int desc, id desc"); + "sort", "float asc, "+intsort+" desc, id desc"); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==10" ,"/response/start==0" @@ -338,7 +401,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { params = params("q", "id:3 id:7", "rows","111", "fl", "id", - "sort", "int asc, id asc"); + "sort", intsort + " asc, id asc"); cursorMark = assertCursor(req(params, CURSOR_MARK_PARAM, cursorMark) ,"/response/numFound==2" ,"/response/start==0" @@ -367,7 +430,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { ids = assertFullWalkNoDups(9, params("q", "*:*", "rows", "3", "fq", "-id:6", - "sort", "float desc, id asc, int asc")); + "sort", "float desc, id asc, "+intsort+" asc")); assertEquals(9, ids.size()); assertFalse("matched on id:6 unexpectedly", ids.exists(6)); ids = assertFullWalkNoDups(9, params("q", "float:[0 TO *] int:7 id:6", @@ -451,7 +514,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { assertU(adoc("id", "3", "str", "a", "float", "11.1", "int", "3")); assertU(commit()); - final Collection allFieldNames = getAllFieldNames(); + final Collection allFieldNames = getAllSortFieldNames(); final SolrInfoMBean filterCacheStats = h.getCore().getInfoRegistry().get("filterCache"); @@ -488,7 +551,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 { /** randomized testing of a non-trivial number of docs using assertFullWalkNoDups */ public void testRandomSortsOnLargeIndex() throws Exception { - final Collection allFieldNames = getAllFieldNames(); + final Collection allFieldNames = getAllSortFieldNames(); final int initialDocs = _TestUtil.nextInt(random(),100,200); final int totalDocs = atLeast(5000); @@ -555,16 +618,42 @@ public class CursorPagingTest extends SolrTestCaseJ4 { } /** - * An immutable list of the fields in the schema (excluding _version_) in a + * An immutable list of the fields in the schema that can be used for sorting, * deterministically random order. */ - private List getAllFieldNames() { + private List getAllSortFieldNames() { + return pruneAndDeterministicallySort + (h.getCore().getLatestSchema().getFields().keySet()); + } + + + /** + *

+ * Given a list of field names in the schema, returns an immutable list in + * deterministically random order with the following things removed: + *

+ *
    + *
  • _version_ is removed
    • + *
  • dv_last and dv_first fields are removed + * if the codec doesn't support them
    • + *
+ * @see #defaultCodecSupportsMissingDocValues + */ + public static List pruneAndDeterministicallySort(Collection raw) { + + final boolean prune_dv_missing = ! defaultCodecSupportsMissingDocValues(); + ArrayList names = new ArrayList(37); - for (String f : h.getCore().getLatestSchema().getFields().keySet()) { - if (! f.equals("_version_")) { - names.add(f); + for (String f : raw) { + if (f.equals("_version_")) { + continue; } + if (prune_dv_missing && (f.endsWith("_dv_last") || f.endsWith("_dv_first")) ) { + continue; + } + names.add(f); } + Collections.sort(names); Collections.shuffle(names,random()); return Collections.unmodifiableList(names); @@ -628,9 +717,9 @@ public class CursorPagingTest extends SolrTestCaseJ4 { } assertU(commit()); - Collection allFieldNames = getAllFieldNames(); + Collection allFieldNames = getAllSortFieldNames(); String[] fieldNames = new String[allFieldNames.size()]; - getAllFieldNames().toArray(fieldNames); + allFieldNames.toArray(fieldNames); String f = fieldNames[_TestUtil.nextInt(random(), 0, fieldNames.length - 1)]; String order = 0 == _TestUtil.nextInt(random(), 0, 1) ? " asc" : " desc"; String sort = f + order + (f.equals("id") ? "" : ", id" + order); diff --git a/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java b/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java index 3f2a79c3ee1..d279d6c7b97 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java @@ -149,7 +149,10 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { String cursorMark = CURSOR_MARK_START; SolrParams params = null; QueryResponse rsp = null; - + + final String intsort = "int" + (random().nextBoolean() ? "" : "_dv"); + final String intmissingsort = defaultCodecSupportsMissingDocValues() ? intsort : "int"; + // trivial base case: ensure cursorMark against an empty index doesn't blow up cursorMark = CURSOR_MARK_START; params = params("q", "*:*", @@ -173,7 +176,7 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { indexDoc(sdoc("id", "6", "str", "a", "float", "64.5", "int", "7")); indexDoc(sdoc("id", "1", "str", "a", "float", "64.5", "int", "7")); indexDoc(sdoc("id", "4", "str", "a", "float", "11.1", "int", "6")); - indexDoc(sdoc("id", "3", "str", "a", "float", "11.1", "int", "3")); + indexDoc(sdoc("id", "3", "str", "a", "float", "11.1")); // int is missing commit(); // base case: ensure cursorMark that matches no docs doesn't blow up @@ -248,7 +251,7 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { "facet", "true", "facet.field", "str", "json.nl", "map", - "sort", "int asc, id asc"); + "sort", intsort + " asc, id asc"); rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); assertNumFound(8, rsp); assertStartsAt(0, rsp); @@ -282,6 +285,70 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { assertEquals("no more docs, but cursorMark has changed", cursorMark, assertHashNextCursorMark(rsp)); + // int missing first sort with dups, id tie breaker + cursorMark = CURSOR_MARK_START; + params = params("q", "-int:2001 -int:4055", + "rows","3", + "fl", "id", + "json.nl", "map", + "sort", intmissingsort + "_first asc, id asc"); + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp, 3, 7, 0); + cursorMark = assertHashNextCursorMark(rsp); + // + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp, 4, 1, 6); + cursorMark = assertHashNextCursorMark(rsp); + // + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp, 9, 2); + cursorMark = assertHashNextCursorMark(rsp); + // + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp); + assertEquals("no more docs, but cursorMark has changed", + cursorMark, assertHashNextCursorMark(rsp)); + + // int missing last sort with dups, id tie breaker + cursorMark = CURSOR_MARK_START; + params = params("q", "-int:2001 -int:4055", + "rows","3", + "fl", "id", + "json.nl", "map", + "sort", intmissingsort + "_last asc, id asc"); + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp, 7, 0, 4); + cursorMark = assertHashNextCursorMark(rsp); + // + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp, 1, 6, 9); + cursorMark = assertHashNextCursorMark(rsp); + // + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp, 2, 3); + cursorMark = assertHashNextCursorMark(rsp); + // + rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); + assertNumFound(8, rsp); + assertStartsAt(0, rsp); + assertDocList(rsp); + assertEquals("no more docs, but cursorMark has changed", + cursorMark, assertHashNextCursorMark(rsp)); + // string sort with dups, id tie breaker cursorMark = CURSOR_MARK_START; params = params("q", "*:*", @@ -312,7 +379,7 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { params = params("q", "*:*", "rows","2", "fl", "id", - "sort", "float asc, int desc, id desc"); + "sort", "float asc, "+intsort+" desc, id desc"); rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); assertNumFound(10, rsp); assertStartsAt(0, rsp); @@ -356,7 +423,7 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { params = params("q", "id:3 id:7", "rows","111", "fl", "id", - "sort", "int asc, id asc"); + "sort", intsort + " asc, id asc"); rsp = query(p(params, CURSOR_MARK_PARAM, cursorMark)); assertNumFound(2, rsp); assertStartsAt(0, rsp); @@ -449,7 +516,7 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { /** randomized testing of a non-trivial number of docs using assertFullWalkNoDups */ public void doRandomSortsOnLargeIndex() throws Exception { - final Collection allFieldNames = getAllFieldNames(); + final Collection allFieldNames = getAllSortFieldNames(); final int numInitialDocs = _TestUtil.nextInt(random(),100,200); final int totalDocs = atLeast(5000); @@ -538,24 +605,20 @@ public class DistribCursorPagingTest extends AbstractFullDistribZkTestBase { /** * Asks the LukeRequestHandler on the control client for a list of the fields in the - * schema (excluding _version_) and then returns the field names in a deterministically - * random order. + * schema and then prunes that list down to just the fields that can be used for sorting, + * and returns them as an immutable list in a deterministically random order. */ - private List getAllFieldNames() throws SolrServerException, IOException { + private List getAllSortFieldNames() throws SolrServerException, IOException { LukeRequest req = new LukeRequest("/admin/luke"); req.setShowSchema(true); NamedList rsp = controlClient.request(req); NamedList fields = (NamedList) ((NamedList)rsp.get("schema")).get("fields"); ArrayList names = new ArrayList(fields.size()); for (Map.Entry item : fields) { - String f = item.getKey(); - if (! f.equals("_version_")) { - names.add(item.getKey()); - } + names.add(item.getKey()); } - Collections.sort(names); - Collections.shuffle(names,random()); - return Collections.unmodifiableList(names); + + return CursorPagingTest.pruneAndDeterministicallySort(names); } /**