SOLR-3177: Enable tagging and excluding filters in StatsComponent via the localParams syntax

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1577976 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -126,6 +126,9 @@ New Features
 * SOLR-5477: Async execution of OverseerCollectionProcessor(CollectionsAPI)
   tasks. (Anshum Gupta)
 
+* SOLR-3177: Enable tagging and excluding filters in StatsComponent via the
+  localParams syntax. (Mathias H., Nikolai Luthman, Vitaliy Zhovtyuk, shalin)
+
 Bug Fixes
 ----------------------
 

solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java

@@ -19,18 +19,24 @@ package org.apache.solr.handler.component;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.lucene.search.*;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.params.StatsParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.UnInvertedField;
 import org.apache.solr.schema.FieldType;
@@ -38,7 +44,10 @@ import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocSet;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryParsing;
 import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SyntaxError;
 
 /**
  * Stats component calculates simple statistics on numeric field values
@@ -62,7 +71,8 @@ public class StatsComponent extends SearchComponent {
       SolrParams params = rb.req.getParams();
       SimpleStats s = new SimpleStats(rb.req,
               rb.getResults().docSet,
-              params );
+              params,
+              rb );
 
       // TODO ???? add this directly to the response, or to the builder?
       rb.rsp.add( "stats", s.getStatsCounts() );
@@ -194,23 +204,98 @@ class SimpleStats {
   /** Searcher to use for all calculations */
   protected SolrIndexSearcher searcher;
   protected SolrQueryRequest req;
+  protected ResponseBuilder rb;
+
+  // per-stats values
+  SolrParams localParams;
+  String statsField;
+  DocSet base;
+  String key;
 
   public SimpleStats(SolrQueryRequest req,
                       DocSet docs,
-                      SolrParams params) {
+                      SolrParams params,
+                      ResponseBuilder rb) {
     this.req = req;
     this.searcher = req.getSearcher();
     this.docs = docs;
     this.params = params;
+    this.rb = rb;
+  }
+
+  protected void parseParams(String param) throws SyntaxError, IOException {
+    localParams = QueryParsing.getLocalParams(param, req.getParams());
+    base = docs;
+    statsField = param;
+    key = param;
+
+    if (localParams == null) return;
+
+    statsField = localParams.get(CommonParams.VALUE);
+
+    // reset set the default key now that localParams have been removed
+    key = statsField;
+
+    // allow explicit set of the key
+    key = localParams.get(CommonParams.OUTPUT_KEY, key);
+
+
+    // figure out if we need a new base DocSet
+    String excludeStr = localParams.get(CommonParams.EXCLUDE);
+    if (excludeStr == null) return;
+
+    Map<?,?> tagMap = (Map<?,?>)req.getContext().get("tags");
+    if (tagMap != null && rb != null) {
+      List<String> excludeTagList = StrUtils.splitSmart(excludeStr,',');
+
+      IdentityHashMap<Query,Boolean> excludeSet = new IdentityHashMap<Query,Boolean>();
+      for (String excludeTag : excludeTagList) {
+        Object olst = tagMap.get(excludeTag);
+        // tagMap has entries of List<String,List<QParser>>, but subject to change in the future
+        if (!(olst instanceof Collection)) continue;
+        for (Object o : (Collection<?>)olst) {
+          if (!(o instanceof QParser)) continue;
+          QParser qp = (QParser)o;
+          excludeSet.put(qp.getQuery(), Boolean.TRUE);
+        }
+      }
+      if (excludeSet.size() == 0) return;
+
+      List<Query> qlist = new ArrayList<Query>();
+
+      // add the base query
+      if (!excludeSet.containsKey(rb.getQuery())) {
+        qlist.add(rb.getQuery());
+      }
+
+      // add the filters
+      if (rb.getFilters() != null) {
+        for (Query q : rb.getFilters()) {
+          if (!excludeSet.containsKey(q)) {
+            qlist.add(q);
+          }
+        }
+      }
+
+      // get the new base docset for this facet
+      this.base = searcher.getDocSet(qlist);
+    }
+
   }
 
   public NamedList<Object> getStatsCounts() throws IOException {
     NamedList<Object> res = new SimpleOrderedMap<>();
-    res.add("stats_fields", getStatsFields());
+
+    try {
+      res.add("stats_fields", getStatsFields());
+    } catch (SyntaxError e) {
+      throw new SolrException(ErrorCode.BAD_REQUEST, e);
+    }
+
     return res;
   }
 
-  public NamedList<Object> getStatsFields() throws IOException {
+  public NamedList<Object> getStatsFields() throws IOException, SyntaxError {
     NamedList<Object> res = new SimpleOrderedMap<>();
     String[] statsFs = params.getParams(StatsParams.STATS_FIELD);
     boolean isShard = params.getBool(ShardParams.IS_SHARD, false);
@@ -218,25 +303,28 @@ class SimpleStats {
       final IndexSchema schema = searcher.getSchema();
       for (String f : statsFs) {
         boolean calcDistinct = params.getFieldBool(f, StatsParams.STATS_CALC_DISTINCT, false);
-        String[] facets = params.getFieldParams(f, StatsParams.STATS_FACET);
+
+        parseParams(f);
+
+        String[] facets = params.getFieldParams(key, StatsParams.STATS_FACET);
         if (facets == null) {
           facets = new String[0]; // make sure it is something...
         }
-        SchemaField sf = schema.getField(f);
+        SchemaField sf = schema.getField(statsField);
         FieldType ft = sf.getType();
         NamedList<?> stv;
 
         if (sf.multiValued() || ft.multiValuedFieldCache()) {
           //use UnInvertedField for multivalued fields
-          UnInvertedField uif = UnInvertedField.getUnInvertedField(f, searcher);
+          UnInvertedField uif = UnInvertedField.getUnInvertedField(statsField, searcher);
           stv = uif.getStats(searcher, docs, calcDistinct, facets).getStatsValues();
         } else {
-          stv = getFieldCacheStats(f, calcDistinct, facets);
+          stv = getFieldCacheStats(statsField, calcDistinct, facets);
         }
         if (isShard == true || (Long) stv.get("count") > 0) {
-          res.add(f, stv);
+          res.add(key, stv);
         } else {
-          res.add(f, null);
+          res.add(key, null);
         }
       }
     }
@@ -263,7 +351,7 @@ class SimpleStats {
 
     final Iterator<AtomicReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
     AtomicReaderContext ctx = null;
-    for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
+    for (DocIterator docsIt = base.iterator(); docsIt.hasNext(); ) {
       final int doc = docsIt.nextDoc();
       if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
         // advance

solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java

@@ -395,4 +395,36 @@ public class StatsComponentTest extends AbstractSolrTestCase {
               400);
 
   }
+
+  //SOLR-3177
+  public void testStatsExcludeFilterQuery() throws Exception {
+    SolrCore core = h.getCore();
+    assertU(adoc("id", "1"));
+    assertU(adoc("id", "2"));
+    assertU(adoc("id", "3"));
+    assertU(adoc("id", "4"));
+    assertU(commit());
+
+    Map<String, String> args = new HashMap<String, String>();
+    args.put(CommonParams.Q, "*:*");
+    args.put(StatsParams.STATS, "true");
+    args.put(StatsParams.STATS_FIELD, "{!ex=id}id");
+    args.put("fq", "{!tag=id}id:[2 TO 3]");
+    SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
+
+    assertQ("test exluding filter query", req
+            , "//lst[@name='id']/double[@name='min'][.='1.0']"
+            , "//lst[@name='id']/double[@name='max'][.='4.0']");
+
+    args = new HashMap<String, String>();
+    args.put(CommonParams.Q, "*:*");
+    args.put(StatsParams.STATS, "true");
+    args.put(StatsParams.STATS_FIELD, "{!key=id2}id");
+    args.put("fq", "{!tag=id}id:[2 TO 3]");
+    req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
+
+    assertQ("test rename field", req
+            , "//lst[@name='id2']/double[@name='min'][.='2.0']"
+            , "//lst[@name='id2']/double[@name='max'][.='3.0']");
+  }
 }