From a928e4b40652cad760cf2d596db08370c07dfc2f Mon Sep 17 00:00:00 2001 From: nknize Date: Tue, 9 Feb 2016 14:13:56 -0600 Subject: [PATCH 1/3] LUCENE-7019: add two-phase iteration to GeoPointTermQueryConstantScoreWrapper --- lucene/CHANGES.txt | 10 +++ ...GeoPointTermQueryConstantScoreWrapper.java | 88 +++++++++++-------- 2 files changed, 60 insertions(+), 38 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index b729f770f72..e86b0aecada 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -294,6 +294,16 @@ Other TestSortingMergePolicy now extend it, TestUpgradeIndexMergePolicy added) (Christine Poerschke) +======================= Lucene 5.4.2 ======================= + +Bug Fixes + +* LUCENE-7018: Fix GeoPointTermQueryConstantScoreWrapper to add document on + first GeoPointField match. (Nick Knize) + +* LUCENE-7019: add two-phase iteration to GeoPointTermQueryConstantScoreWrapper. + (Robert Muir via Nick Knize) + ======================= Lucene 5.4.1 ======================= Bug Fixes diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/search/GeoPointTermQueryConstantScoreWrapper.java b/lucene/spatial/src/java/org/apache/lucene/spatial/search/GeoPointTermQueryConstantScoreWrapper.java index 1097addb3b7..46a17833e32 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/search/GeoPointTermQueryConstantScoreWrapper.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/search/GeoPointTermQueryConstantScoreWrapper.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.Terms; -import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSet; @@ -31,8 +30,12 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BitSet; import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.SparseFixedBitSet; import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonUnhashLat; import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonUnhashLon; @@ -74,67 +77,76 @@ final class GeoPointTermQueryConstantScoreWrapper Date: Wed, 10 Feb 2016 01:06:56 +0300 Subject: [PATCH 2/3] SOLR-8466: facet.method=uif for UnInvertedField faceting, like it was with 'fc' earlier. --- solr/CHANGES.txt | 4 + .../org/apache/solr/request/SimpleFacets.java | 76 +++++++++- .../solr/search/facet/FacetProcessor.java | 22 +++ .../apache/solr/TestDistributedSearch.java | 15 +- .../org/apache/solr/TestRandomDVFaceting.java | 9 +- .../apache/solr/request/SimpleFacetsTest.java | 136 +++++++++++------- .../org/apache/solr/request/TestFaceting.java | 89 ++++++++---- .../solr/common/params/FacetParams.java | 5 + 8 files changed, 271 insertions(+), 85 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d37ce33a5e8..c310376e85d 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -146,6 +146,10 @@ New Features * SOLR-8502: Improve Solr JDBC Driver to support SQL Clients like DBVisualizer (Kevin Risden, Joel Bernstein) +* SOLR-8466: adding facet.method=uif to bring back UnInvertedField faceting which is used to work on + facet.method=fc. It's more performant for rarely changing indexes. Note: it ignores prefix and contains yet. + (Jamie Johnson via Mikhail Khludnev) + Bug Fixes ---------------------- * SOLR-8386: Add field option in the new admin UI schema page loads up even when no schemaFactory has been diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index e13cd75be92..9f409cc87a1 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -69,6 +69,7 @@ import org.apache.solr.search.QueryParsing; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SortedIntDocSet; import org.apache.solr.search.SyntaxError; +import org.apache.solr.search.facet.FacetProcessor; import org.apache.solr.search.grouping.GroupingSpecification; import org.apache.solr.util.BoundedTreeSet; import org.apache.solr.util.DefaultSolrThreadFactory; @@ -77,6 +78,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; @@ -360,7 +362,7 @@ public class SimpleFacets { } enum FacetMethod { - ENUM, FC, FCS; + ENUM, FC, FCS, UIF; } /** @@ -422,6 +424,8 @@ public class SimpleFacets { method = FacetMethod.FCS; } else if (FacetParams.FACET_METHOD_fc.equals(methodStr)) { method = FacetMethod.FC; + } else if(FacetParams.FACET_METHOD_uif.equals(methodStr)) { + method = FacetMethod.UIF; } if (method == FacetMethod.ENUM && TrieField.getMainValuePrefix(ft) != null) { @@ -485,6 +489,73 @@ public class SimpleFacets { counts = ps.getFacetCounts(executor); } break; + case UIF: + + //Emulate the JSON Faceting structure so we can use the same parsing classes + Map jsonFacet = new HashMap<>(13); + jsonFacet.put("type", "terms"); + jsonFacet.put("field", field); + jsonFacet.put("offset", offset); + jsonFacet.put("limit", limit); + jsonFacet.put("mincount", mincount); + jsonFacet.put("missing", missing); + + if (prefix!=null) { + // presumably it supports single-value, but at least now returns wrong results on multi-value + throw new SolrException ( + SolrException.ErrorCode.BAD_REQUEST, + FacetParams.FACET_PREFIX+"="+prefix+ + " are not supported by "+FacetParams.FACET_METHOD+"="+FacetParams.FACET_METHOD_uif+ + " for field:"+ field + //jsonFacet.put("prefix", prefix); + ); + } + jsonFacet.put("numBuckets", params.getFieldBool(field, "numBuckets", false)); + jsonFacet.put("allBuckets", params.getFieldBool(field, "allBuckets", false)); + jsonFacet.put("method", "uif"); + jsonFacet.put("cacheDf", 0); + jsonFacet.put("perSeg", false); + + final String sortVal; + switch(sort){ + case FacetParams.FACET_SORT_COUNT_LEGACY: + sortVal = FacetParams.FACET_SORT_COUNT; + break; + case FacetParams.FACET_SORT_INDEX_LEGACY: + sortVal = FacetParams.FACET_SORT_INDEX; + break; + default: + sortVal = sort; + } + jsonFacet.put("sort", sortVal ); + + Map topLevel = new HashMap<>(); + topLevel.put(field, jsonFacet); + + topLevel.put("processEmpty", true); + + FacetProcessor fproc = FacetProcessor.createProcessor(rb.req, topLevel, // rb.getResults().docSet + docs ); + //TODO do we handle debug? Should probably already be handled by the legacy code + fproc.process(); + + //Go through the response to build the expected output for SimpleFacets + Object res = fproc.getResponse(); + counts = new NamedList(); + if(res != null) { + SimpleOrderedMap som = (SimpleOrderedMap)res; + SimpleOrderedMap asdf = (SimpleOrderedMap) som.get(field); + + List> buckets = (List>)asdf.get("buckets"); + for(SimpleOrderedMap b : buckets) { + counts.add(b.get("val").toString(), (Integer)b.get("count")); + } + if(missing) { + SimpleOrderedMap missingCounts = (SimpleOrderedMap) asdf.get("missing"); + counts.add(null, (Integer)missingCounts.get("count")); + } + } + break; case FC: counts = DocValuesFacets.getCounts(searcher, docs, field, offset,limit, mincount, missing, sort, prefix, contains, ignoreCase); break; @@ -958,5 +1029,4 @@ public class SimpleFacets { public ResponseBuilder getResponseBuilder() { return rb; } -} - +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java index ba1aa44c9ca..37013b0a330 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java @@ -33,12 +33,14 @@ import org.apache.lucene.search.Query; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.BitDocSet; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocSet; import org.apache.solr.search.QParser; +import org.apache.solr.search.QueryContext; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SyntaxError; import org.apache.solr.util.RTimer; @@ -60,6 +62,26 @@ public class FacetProcessor { public void process() throws IOException { handleDomainChanges(); } + + /** factory method for invoking json facet framework as whole */ + public static FacetProcessor createProcessor(SolrQueryRequest req, + Map params, DocSet docs){ + FacetParser parser = new FacetTopParser(req); + FacetRequest facetRequest = null; + try { + facetRequest = parser.parse(params); + } catch (SyntaxError syntaxError) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, syntaxError); + } + + FacetContext fcontext = new FacetContext(); + fcontext.base = docs; + fcontext.req = req; + fcontext.searcher = req.getSearcher(); + fcontext.qcontext = QueryContext.newContext(fcontext.searcher); + + return facetRequest.createFacetProcessor(fcontext); + } protected void handleDomainChanges() throws IOException { if (freq.domain == null) return; diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java index 7e0751410a9..b6750c27072 100644 --- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java @@ -409,14 +409,14 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { ,"facet.field",t1); // test filter tagging, facet exclusion, and naming (multi-select facet support) - query("q","*:*", "rows",0, "facet","true", "facet.query","{!key=myquick}quick", "facet.query","{!key=myall ex=a}all", "facet.query","*:*" + queryAndCompareUIF("q","*:*", "rows",0, "facet","true", "facet.query","{!key=myquick}quick", "facet.query","{!key=myall ex=a}all", "facet.query","*:*" ,"facet.field","{!key=mykey ex=a}"+t1 ,"facet.field","{!key=other ex=b}"+t1 ,"facet.field","{!key=again ex=a,b}"+t1 ,"facet.field",t1 ,"fq","{!tag=a}id:[1 TO 7]", "fq","{!tag=b}id:[3 TO 9]" ); - query("q", "*:*", "facet", "true", "facet.field", "{!ex=t1}SubjectTerms_mfacet", "fq", "{!tag=t1}SubjectTerms_mfacet:(test 1)", "facet.limit", "10", "facet.mincount", "1"); + queryAndCompareUIF("q", "*:*", "facet", "true", "facet.field", "{!ex=t1}SubjectTerms_mfacet", "fq", "{!tag=t1}SubjectTerms_mfacet:(test 1)", "facet.limit", "10", "facet.mincount", "1"); // test field that is valid in schema but missing in all shards query("q","*:*", "rows",100, "facet","true", "facet.field",missingField, "facet.mincount",2); @@ -1051,6 +1051,17 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase { "stats.facet", fieldName); } + /** comparing results with facet.method=uif */ + private void queryAndCompareUIF(Object ... params) throws Exception { + final QueryResponse expect = query(params); + + final Object[] newParams = Arrays.copyOf(params, params.length+2); + newParams[newParams.length-2] = "facet.method"; + newParams[newParams.length-1] = "uif"; + final QueryResponse uifResult = query(newParams); + compareResponses(expect, uifResult); + } + protected void checkMinCountsField(List counts, Object[] pairs) { assertEquals("There should be exactly " + pairs.length / 2 + " returned counts. There were: " + counts.size(), counts.size(), pairs.length / 2); assertTrue("Variable len param must be an even number, it was: " + pairs.length, (pairs.length % 2) == 0); diff --git a/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java b/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java index 8ac0bcc4ed1..7decfce45a4 100644 --- a/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java +++ b/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java @@ -144,8 +144,8 @@ public class TestRandomDVFaceting extends SolrTestCaseJ4 { // NOTE: dv is not a "real" facet.method. when we see it, we facet on the dv field (*_dv) // but alias the result back as if we faceted on the regular indexed field for comparisons. - List multiValuedMethods = Arrays.asList(new String[]{"enum","fc","dv"}); - List singleValuedMethods = Arrays.asList(new String[]{"enum","fc","fcs","dv"}); + List multiValuedMethods = Arrays.asList(new String[]{"enum","fc","dv","uif"}); + List singleValuedMethods = Arrays.asList(new String[]{"enum","fc","fcs","dv","uif"}); void doFacetTests(FldType ftype) throws Exception { @@ -215,6 +215,9 @@ public class TestRandomDVFaceting extends SolrTestCaseJ4 { List methods = multiValued ? multiValuedMethods : singleValuedMethods; List responses = new ArrayList<>(methods.size()); for (String method : methods) { + if (method.equals("uif") && params.get("facet.prefix")!=null) { + continue; // it's not supported there + } if (method.equals("dv")) { params.set("facet.field", "{!key="+facet_field+"}"+facet_field+"_dv"); params.set("facet.method",(String) null); @@ -238,7 +241,7 @@ public class TestRandomDVFaceting extends SolrTestCaseJ4 { **/ if (validate) { - for (int i=1; i Date: Wed, 10 Feb 2016 01:21:18 +0300 Subject: [PATCH 3/3] SOLR-8466: fixing CHANGES.txt moving to 5.5.0 Features. --- solr/CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c310376e85d..a6ef8343b8a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -146,10 +146,6 @@ New Features * SOLR-8502: Improve Solr JDBC Driver to support SQL Clients like DBVisualizer (Kevin Risden, Joel Bernstein) -* SOLR-8466: adding facet.method=uif to bring back UnInvertedField faceting which is used to work on - facet.method=fc. It's more performant for rarely changing indexes. Note: it ignores prefix and contains yet. - (Jamie Johnson via Mikhail Khludnev) - Bug Fixes ---------------------- * SOLR-8386: Add field option in the new admin UI schema page loads up even when no schemaFactory has been @@ -406,6 +402,10 @@ New Features * SOLR-8648: DELETESTATUS API for selective deletion and flushing of stored async collection API responses. (Anshum Gupta) +* SOLR-8466: adding facet.method=uif to bring back UnInvertedField faceting which is used to work on + facet.method=fc. It's more performant for rarely changing indexes. Note: it ignores prefix and contains yet. + (Jamie Johnson via Mikhail Khludnev) + Bug Fixes ----------------------