Merging with trunk.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/solr7787@1691514 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -259,6 +259,9 @@ Bug fixes
 * LUCENE-6680: Preserve two suggestions that have same key and weight but
   different payloads (Arcadius Ahouansou via Mike McCandless)
 
+* LUCENE-6681: SortingMergePolicy must override MergePolicy.size(...).
+  (Christine Poerschke via Adrien Grand)
+
 Changes in Runtime Behavior
 
 * LUCENE-6501: The subreader structure in ParallelCompositeReader

lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java

@@ -260,6 +260,11 @@ public final class SortingMergePolicy extends MergePolicy {
     return in.useCompoundFile(segments, newSegment, writer);
   }
 
+  @Override
+  protected long size(SegmentCommitInfo info, IndexWriter writer) throws IOException {
+    return in.size(info, writer);
+  }
+
   @Override
   public String toString() {
     return "SortingMergePolicy(" + in + ", sorter=" + sorter + ")";

lucene/misc/src/test/org/apache/lucene/index/TestSortingMergePolicy.java

@@ -18,6 +18,8 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -192,4 +194,15 @@ public class TestSortingMergePolicy extends LuceneTestCase {
     }
   }
 
+  public void testMethodsOverridden() throws Exception {
+    for (Method m : MergePolicy.class.getDeclaredMethods()) {
+      if (Modifier.isFinal(m.getModifiers())) continue;
+      try {
+        SortingMergePolicy.class.getDeclaredMethod(m.getName(),  m.getParameterTypes());
+      } catch (NoSuchMethodException e) {
+        fail("SortingMergePolicy needs to override '"+m+"'");
+      }
+    }
+  }
+
 }

solr/CHANGES.txt

@@ -332,6 +332,12 @@ Other Changes
 * SOLR-7703: Authentication plugin is now loaded using the RessourceLoader.
   (Avi Digmi via Anshum Gupta)
 
+* SOLR-7800: JSON Facet API: the avg() facet function now skips missing values
+  rather than treating them as a 0 value.  The def() function can be used to
+  treat missing values as 0 if that is desired.
+  Example:  facet:{ mean:"avg(def(myfield,0))" }
+
+
 ==================  5.2.1 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release

solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java

@@ -348,9 +348,11 @@ class AvgSlotAcc extends DoubleFuncSlotAcc {
 
   @Override
   public void collect(int doc, int slotNum) {
-    double val = values.doubleVal(doc);  // todo: worth trying to share this value across multiple stats that need it?
+    double val = values.doubleVal(doc);
-    result[slotNum] += val;
+    if (val != 0 || values.exists(doc)) {
-    counts[slotNum] += 1;
+      result[slotNum] += val;
+      counts[slotNum] += 1;
+    }
   }
 
   private double avg(double tot, int count) {

solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java

@@ -737,13 +737,13 @@ public class TestJsonFacets extends SolrTestCaseHS {
 
     // stats at top level
     client.testJQ(params(p, "q", "*:*"
-            , "json.facet", "{ sum1:'sum(${num_d})', sumsq1:'sumsq(${num_d})', avg1:'avg(${num_d})', min1:'min(${num_d})', max1:'max(${num_d})'" +
+            , "json.facet", "{ sum1:'sum(${num_d})', sumsq1:'sumsq(${num_d})', avg1:'avg(${num_d})', avg2:'avg(def(${num_d},0))', min1:'min(${num_d})', max1:'max(${num_d})'" +
                 ", numwhere:'unique(${where_s})', unique_num_i:'unique(${num_i})', unique_num_d:'unique(${num_d})', unique_date:'unique(${date})'" +
                 ", where_hll:'hll(${where_s})', hll_num_i:'hll(${num_i})', hll_num_d:'hll(${num_d})', hll_date:'hll(${date})'" +
                 ", med:'percentile(${num_d},50)', perc:'percentile(${num_d},0,50.0,100)' }"
         )
         , "facets=={ 'count':6, " +
-            "sum1:3.0, sumsq1:247.0, avg1:0.5, min1:-9.0, max1:11.0" +
+            "sum1:3.0, sumsq1:247.0, avg1:0.6, avg2:0.5, min1:-9.0, max1:11.0" +
             ", numwhere:2, unique_num_i:4, unique_num_d:5, unique_date:5" +
             ", where_hll:2, hll_num_i:4, hll_num_d:5, hll_date:5" +
             ", med:2.0, perc:[-9.0,2.0,11.0]  }"

solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java

@@ -49,6 +49,7 @@ public class QueryResponse extends SolrResponseBase
   private NamedList<Object> _highlightingInfo = null;
   private NamedList<Object> _spellInfo = null;
   private List<NamedList<Object>> _clusterInfo = null;
+  private Map<String,NamedList<Object>> _suggestInfo = null;
   private NamedList<Object> _statsInfo = null;
   private NamedList<NamedList<Number>> _termsInfo = null;
   private String _cursorMarkNext = null;
@@ -78,6 +79,9 @@ public class QueryResponse extends SolrResponseBase
   // Clustering Response
   private ClusteringResponse _clusterResponse = null;
 
+  // Suggester Response
+  private SuggesterResponse _suggestResponse = null;
+
   // Terms Response
   private TermsResponse _termsResponse = null;
   
@@ -153,6 +157,10 @@ public class QueryResponse extends SolrResponseBase
         _clusterInfo = (ArrayList<NamedList<Object>>) res.getVal(i);
         extractClusteringInfo(_clusterInfo);
       }
+      else if ( "suggest".equals( n ) )  {
+        _suggestInfo = (Map<String,NamedList<Object>>) res.getVal( i );
+        extractSuggesterInfo(_suggestInfo);
+      }
       else if ( "stats".equals( n ) )  {
         _statsInfo = (NamedList<Object>) res.getVal( i );
         extractStatsInfo( _statsInfo );
@@ -176,6 +184,10 @@ public class QueryResponse extends SolrResponseBase
     _clusterResponse = new ClusteringResponse(clusterInfo);
   }
 
+  private void extractSuggesterInfo(Map<String, NamedList<Object>> suggestInfo) {
+    _suggestResponse = new SuggesterResponse(suggestInfo);
+  }
+
   private void extractTermsInfo(NamedList<NamedList<Number>> termsInfo) {
     _termsResponse = new TermsResponse(termsInfo);
   }
@@ -553,6 +565,10 @@ public class QueryResponse extends SolrResponseBase
     return _clusterResponse;
   }
 
+  public SuggesterResponse getSuggesterResponse() {
+    return _suggestResponse;
+  }
+
   public TermsResponse getTermsResponse() {
     return _termsResponse;
   }

solr/solrj/src/java/org/apache/solr/client/solrj/response/SuggesterResponse.java

@@ -0,0 +1,87 @@
+package org.apache.solr.client.solrj.response;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+
+/**
+ * Encapsulates responses from the Suggester Component
+ */
+public class SuggesterResponse {
+
+  private static final String SUGGESTIONS_NODE_NAME = "suggestions";
+  private static final String TERM_NODE_NAME = "term";
+  private static final String WEIGHT_NODE_NAME = "weight";
+  private static final String PAYLOAD_NODE_NAME = "payload";
+
+  private final Map<String, List<Suggestion>> suggestionsPerDictionary = new LinkedHashMap<>();
+
+  public SuggesterResponse(Map<String, NamedList<Object>> suggestInfo) {
+    for (Map.Entry<String, NamedList<Object>> entry : suggestInfo.entrySet()) {
+      SimpleOrderedMap suggestionsNode = (SimpleOrderedMap) entry.getValue().getVal(0);
+      List<SimpleOrderedMap> suggestionListToParse;
+      List<Suggestion> suggestionList = new LinkedList<>();
+      if (suggestionsNode != null) {
+
+        suggestionListToParse = (List<SimpleOrderedMap>) suggestionsNode.get(SUGGESTIONS_NODE_NAME);
+        for (SimpleOrderedMap suggestion : suggestionListToParse) {
+          String term = (String) suggestion.get(TERM_NODE_NAME);
+          long weight = (long) suggestion.get(WEIGHT_NODE_NAME);
+          String payload = (String) suggestion.get(PAYLOAD_NODE_NAME);
+
+          Suggestion parsedSuggestion = new Suggestion(term, weight, payload);
+          suggestionList.add(parsedSuggestion);
+        }
+        suggestionsPerDictionary.put(entry.getKey(), suggestionList);
+      }
+    }
+  }
+
+  /**
+   * get the suggestions provided by each
+   *
+   * @return a Map dictionary name : List of Suggestion
+   */
+  public Map<String, List<Suggestion>> getSuggestions() {
+    return suggestionsPerDictionary;
+  }
+
+  /**
+   * This getter is lazily initialized and returns a simplified map dictionary : List of suggested terms
+   * This is useful for simple use cases when you simply need the suggested terms and no weight or payload
+   *
+   * @return a Map dictionary name : List of suggested terms
+   */
+  public Map<String, List<String>> getSuggestedTerms() {
+    Map<String, List<String>> suggestedTermsPerDictionary = new LinkedHashMap<>();
+    for (Map.Entry<String, List<Suggestion>> entry : suggestionsPerDictionary.entrySet()) {
+      List<Suggestion> suggestions = entry.getValue();
+      List<String> suggestionTerms = new LinkedList<String>();
+      for (Suggestion s : suggestions) {
+        suggestionTerms.add(s.getTerm());
+      }
+      suggestedTermsPerDictionary.put(entry.getKey(), suggestionTerms);
+    }
+    return suggestedTermsPerDictionary;
+  }
+}

solr/solrj/src/java/org/apache/solr/client/solrj/response/Suggestion.java

@@ -0,0 +1,63 @@
+package org.apache.solr.client.solrj.response;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This class models a Suggestion coming from Solr Suggest Component.
+ * It is a direct mapping fo the Json object Solr is returning.
+ */
+public class Suggestion {
+    private String term;
+    private long weight;
+    private String payload;
+
+    public Suggestion(String term, long weight, String payload) {
+        this.term = term;
+        this.weight = weight;
+        this.payload = payload;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof Suggestion)) return false;
+
+        Suggestion that = (Suggestion) o;
+
+        return payload.equals(that.payload) && term.equals(that.term);
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = term.hashCode();
+        result = 31 * result + payload.hashCode();
+        return result;
+    }
+
+    public String getTerm() {
+        return term;
+    }
+
+    public long getWeight() {
+        return weight;
+    }
+
+    public String getPayload() {
+        return payload;
+    }
+
+}

solr/solrj/src/test/org/apache/solr/client/solrj/response/TestSuggesterResponse.java

@@ -0,0 +1,128 @@
+package org.apache.solr.client.solrj.response;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.SolrJettyTestBase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test for SuggesterComponent's response in Solrj
+ *
+ */
+public class TestSuggesterResponse extends SolrJettyTestBase {
+  @BeforeClass
+  public static void beforeTest() throws Exception {
+    initCore();
+  }
+
+  static String field = "cat";
+
+  @Test
+  public void testSuggesterResponseObject() throws Exception {
+    getSolrClient();
+    addSampleDocs();
+
+    SolrQuery query = new SolrQuery("*:*");
+    query.set(CommonParams.QT, "/suggest");
+    query.set("suggest.dictionary", "mySuggester");
+    query.set("suggest.q", "Com");
+    query.set("suggest.build", true);
+    QueryRequest request = new QueryRequest(query);
+    QueryResponse queryResponse = request.process(client);
+    SuggesterResponse response = queryResponse.getSuggesterResponse();
+    Map<String, List<Suggestion>> dictionary2suggestions = response.getSuggestions();
+    assertTrue(dictionary2suggestions.keySet().contains("mySuggester"));
+
+    List<Suggestion> mySuggester = dictionary2suggestions.get("mySuggester");
+    assertEquals("Computational framework", mySuggester.get(0).getTerm());
+    assertEquals(0, mySuggester.get(0).getWeight());
+    assertEquals("", mySuggester.get(0).getPayload());
+    assertEquals("Computer", mySuggester.get(1).getTerm());
+    assertEquals(0, mySuggester.get(1).getWeight());
+    assertEquals("", mySuggester.get(1).getPayload());
+  }
+
+  @Test
+  public void testSuggesterResponseTerms() throws Exception {
+    getSolrClient();
+    addSampleDocs();
+
+    SolrQuery query = new SolrQuery("*:*");
+    query.set(CommonParams.QT, "/suggest");
+    query.set("suggest.dictionary", "mySuggester");
+    query.set("suggest.q", "Com");
+    query.set("suggest.build", true);
+    QueryRequest request = new QueryRequest(query);
+    QueryResponse queryResponse = request.process(client);
+    SuggesterResponse response = queryResponse.getSuggesterResponse();
+    Map<String, List<String>> dictionary2suggestions = response.getSuggestedTerms();
+    assertTrue(dictionary2suggestions.keySet().contains("mySuggester"));
+
+    List<String> mySuggester = dictionary2suggestions.get("mySuggester");
+    assertEquals("Computational framework", mySuggester.get(0));
+    assertEquals("Computer", mySuggester.get(1));
+  }
+
+  @Test
+  public void testEmptySuggesterResponse() throws Exception {
+    getSolrClient();
+    addSampleDocs();
+
+    SolrQuery query = new SolrQuery("*:*");
+    query.set(CommonParams.QT, "/suggest");
+    query.set("suggest.dictionary", "mySuggester");
+    query.set("suggest.q", "Empty");
+    query.set("suggest.build", true);
+    QueryRequest request = new QueryRequest(query);
+    QueryResponse queryResponse = request.process(client);
+    SuggesterResponse response = queryResponse.getSuggesterResponse();
+    Map<String, List<String>> dictionary2suggestions = response.getSuggestedTerms();
+    assertTrue(dictionary2suggestions.keySet().contains("mySuggester"));
+
+    List<String> mySuggester = dictionary2suggestions.get("mySuggester");
+    assertEquals(0, mySuggester.size());
+  }
+
+  private void addSampleDocs() throws SolrServerException, IOException {
+    client.deleteByQuery("*:*");
+    client.commit(true, true);
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.setField("id", "111");
+    doc.setField(field, "Computer");
+    SolrInputDocument doc2 = new SolrInputDocument();
+    doc2.setField("id", "222");
+    doc2.setField(field, "Computational framework");
+    SolrInputDocument doc3 = new SolrInputDocument();
+    doc3.setField("id", "333");
+    doc3.setField(field, "Laptop");
+    client.add(doc);
+    client.add(doc2);
+    client.add(doc3);
+    client.commit(true, true);
+  }
+
+}