catch up with trunk

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/LUCENE2793@1137636 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -434,6 +434,10 @@ New features
   need to lookup by that field or perform deletions against it, for
   example in a near-real-time setting. (Mike McCandless)
 
+* SOLR-2533: Added support for rewriting Sort and SortFields using an
+  IndexSearcher.  SortFields can have SortField.REWRITEABLE type which
+  requires they are rewritten before they are used. (Chris Male)
+
 Optimizations
 
 * LUCENE-2588: Don't store unnecessary suffixes when writing the terms

lucene/src/java/org/apache/lucene/search/Sort.java

@@ -17,6 +17,7 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
+import java.io.IOException;
 import java.util.Arrays;
 
 
@@ -149,6 +150,30 @@ public class Sort {
     return fields;
   }
 
+  /**
+   * Rewrites the SortFields in this Sort, returning a new Sort if any of the fields
+   * changes during their rewriting.
+   *
+   * @param searcher IndexSearcher to use in the rewriting
+   * @return {@code this} if the Sort/Fields have not changed, or a new Sort if there
+   *        is a change
+   * @throws IOException Can be thrown by the rewriting
+   * @lucene.experimental
+   */
+  public Sort rewrite(IndexSearcher searcher) throws IOException {
+    boolean changed = false;
+    
+    SortField[] rewrittenSortFields = new SortField[fields.length];
+    for (int i = 0; i < fields.length; i++) {
+      rewrittenSortFields[i] = fields[i].rewrite(searcher);
+      if (fields[i] != rewrittenSortFields[i]) {
+        changed = true;
+      }
+    }
+
+    return (changed) ? new Sort(rewrittenSortFields) : this;
+  }
+
   @Override
   public String toString() {
     StringBuilder buffer = new StringBuilder();

lucene/src/java/org/apache/lucene/search/SortField.java

@@ -86,10 +86,14 @@ public class SortField {
    * This is typically slower than {@link #STRING}, which
    * uses ordinals to do the sorting. */
   public static final int STRING_VAL = 11;
-  
+
   /** Sort use byte[] index values. */
   public static final int BYTES = 12;
-  
+
+  /** Force rewriting of SortField using {@link SortField#rewrite(IndexSearcher)}
+   * before it can be used for sorting */
+  public static final int REWRITEABLE = 13;
+
   /** Represents sorting by document score (relevance). */
   public static final SortField FIELD_SCORE = new SortField(null, SCORE);
 
@@ -475,9 +479,26 @@ public class SortField {
 
     case SortField.STRING_VAL:
       return new FieldComparator.TermValComparator(numHits, field);
+
+    case SortField.REWRITEABLE:
+      throw new IllegalStateException("SortField needs to be rewritten through Sort.rewrite(..) and SortField.rewrite(..)");
         
     default:
       throw new IllegalStateException("Illegal sort type: " + type);
     }
   }
+
+  /**
+   * Rewrites this SortField, returning a new SortField if a change is made.
+   * Subclasses should override this define their rewriting behavior when this
+   * SortField is of type {@link SortField#REWRITEABLE}
+   *
+   * @param searcher IndexSearcher to use during rewriting
+   * @return New rewritten SortField, or {@code this} if nothing has changed.
+   * @throws IOException Can be thrown by the rewriting
+   * @lucene.experimental
+   */
+  public SortField rewrite(IndexSearcher searcher) throws IOException {
+    return this;
+  }
 }

lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java

@@ -148,35 +148,23 @@ public class QueryUtils {
     // we can't put deleted docs before the nested reader, because
     // it will throw off the docIds
     IndexReader[] readers = new IndexReader[] {
-      edge < 0 ? r : emptyReaders[0],
+      edge < 0 ? r : IndexReader.open(makeEmptyIndex(random, 0), true),
-      emptyReaders[0],
+      IndexReader.open(makeEmptyIndex(random, 0), true),
-      new MultiReader(edge < 0 ? emptyReaders[4] : emptyReaders[0],
+      new MultiReader(IndexReader.open(makeEmptyIndex(random, edge < 0 ? 4 : 0), true),
-          emptyReaders[0],
+          IndexReader.open(makeEmptyIndex(random, 0), true),
-          0 == edge ? r : emptyReaders[0]),
+          0 == edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true)),
-      0 < edge ? emptyReaders[0] : emptyReaders[7],
+      IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 7), true),
-      emptyReaders[0],
+      IndexReader.open(makeEmptyIndex(random, 0), true),
-      new MultiReader(0 < edge ? emptyReaders[0] : emptyReaders[5],
+      new MultiReader(IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 5), true),
-          emptyReaders[0],
+          IndexReader.open(makeEmptyIndex(random, 0), true),
-          0 < edge ? r : emptyReaders[0])
+          0 < edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true))
     };
     IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
     out.setSimilarityProvider(s.getSimilarityProvider());
     return out;
   }
-  
-  static final IndexReader[] emptyReaders = new IndexReader[8];
-  static {
-    try {
-      emptyReaders[0] = makeEmptyIndex(new Random(0), 0);
-      emptyReaders[4] = makeEmptyIndex(new Random(0), 4);
-      emptyReaders[5] = makeEmptyIndex(new Random(0), 5);
-      emptyReaders[7] = makeEmptyIndex(new Random(0), 7);
-    } catch (IOException ex) {
-      throw new RuntimeException(ex);
-    }
-  }
 
-  private static IndexReader makeEmptyIndex(Random random, final int numDeletedDocs) 
+  private static Directory makeEmptyIndex(Random random, final int numDeletedDocs) 
     throws IOException {
     Directory d = new MockDirectoryWrapper(random, new RAMDirectory());
       IndexWriter w = new IndexWriter(d, new IndexWriterConfig(
@@ -200,7 +188,8 @@ public class QueryUtils {
       IndexReader r = IndexReader.open(d, true);
       Assert.assertEquals("reader has wrong number of deleted docs", 
                           numDeletedDocs, r.numDeletedDocs());
-      return r;
+      r.close();
+      return d;
   }
 
   /** alternate scorer skipTo(),skipTo(),next(),next(),skipTo(),skipTo(), etc

solr/CHANGES.txt

@@ -145,6 +145,9 @@ New Features
 * SOLR-2417: Add explain info directly to return documents using 
   ?fl=id,[explain] (ryan)
 
+* SOLR-2533: Converted ValueSource.ValueSourceSortField over to new rewriteable Lucene
+  SortFields.  ValueSourceSortField instances must be rewritten before they can be used.
+  This is done by SolrIndexSearcher when necessary. (Chris Male).
   
 
 Optimizations

solr/src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -475,26 +475,7 @@ public class SolrIndexSearcher extends IndexSearcher implements SolrInfoMBean {
 
   /** Returns a weighted sort according to this searcher */
   public Sort weightSort(Sort sort) throws IOException {
-    if (sort == null) return null;
+    return (sort != null) ? sort.rewrite(this) : null;
-    SortField[] sorts = sort.getSort();
-
-    boolean needsWeighting = false;
-    for (SortField sf : sorts) {
-      if (sf instanceof SolrSortField) {
-        needsWeighting = true;
-        break;
-      }
-    }
-    if (!needsWeighting) return sort;
-
-    SortField[] newSorts = Arrays.copyOf(sorts, sorts.length);
-    for (int i=0; i<newSorts.length; i++) {
-      if (newSorts[i] instanceof SolrSortField) {
-        newSorts[i] = ((SolrSortField)newSorts[i]).weight(this);
-      }
-    }
-
-    return new Sort(newSorts);
   }
 
 

solr/src/java/org/apache/solr/search/function/ValueSource.java

@@ -26,8 +26,6 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.index.MultiFields;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.search.SolrSortField;
 
 import java.io.IOException;
 import java.io.Serializable;
@@ -102,20 +100,13 @@ public abstract class ValueSource implements Serializable {
     return new ValueSourceSortField(reverse);
   }
 
-  private static FieldComparatorSource dummyComparator = new FieldComparatorSource() {
+  class ValueSourceSortField extends SortField {
-    @Override
-    public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unweighted use of sort " + fieldname);
-    }
-  };
-
-  class ValueSourceSortField extends SortField implements SolrSortField {
     public ValueSourceSortField(boolean reverse) {
-      super(description(), dummyComparator, reverse);
+      super(description(), SortField.REWRITEABLE, reverse);
     }
 
     @Override
-    public SortField weight(IndexSearcher searcher) throws IOException {
+    public SortField rewrite(IndexSearcher searcher) throws IOException {
       Map context = newContext(searcher);
       createWeight(context, searcher);
       return new SortField(getField(), new ValueSourceComparatorSource(context), getReverse());

solr/src/test/org/apache/solr/search/QueryParsingTest.java

@@ -96,7 +96,7 @@ public class QueryParsingTest extends SolrTestCaseJ4 {
     //test functions
     sort = QueryParsing.parseSort("pow(weight, 2) desc", req);
     flds = sort.getSort();
-    assertEquals(flds[0].getType(), SortField.CUSTOM);
+    assertEquals(flds[0].getType(), SortField.REWRITEABLE);
     //Not thrilled about the fragility of string matching here, but...
     //the value sources get wrapped, so the out field is different than the input
     assertEquals(flds[0].getField(), "pow(float(weight),const(2))");
@@ -104,12 +104,12 @@ public class QueryParsingTest extends SolrTestCaseJ4 {
     //test functions (more deep)
     sort = QueryParsing.parseSort("sum(product(r_f1,sum(d_f1,t_f1,1.0)),a_f1) asc", req);
     flds = sort.getSort();
-    assertEquals(flds[0].getType(), SortField.CUSTOM);
+    assertEquals(flds[0].getType(), SortField.REWRITEABLE);
     assertEquals(flds[0].getField(), "sum(product(float(r_f1),sum(float(d_f1),float(t_f1),const(1.0))),float(a_f1))");
 
     sort = QueryParsing.parseSort("pow(weight,                 2.0)         desc", req);
     flds = sort.getSort();
-    assertEquals(flds[0].getType(), SortField.CUSTOM);
+    assertEquals(flds[0].getType(), SortField.REWRITEABLE);
     //Not thrilled about the fragility of string matching here, but...
     //the value sources get wrapped, so the out field is different than the input
     assertEquals(flds[0].getField(), "pow(float(weight),const(2.0))");
@@ -117,7 +117,7 @@ public class QueryParsingTest extends SolrTestCaseJ4 {
 
     sort = QueryParsing.parseSort("pow(weight, 2.0) desc, weight    desc,   bday    asc", req);
     flds = sort.getSort();
-    assertEquals(flds[0].getType(), SortField.CUSTOM);
+    assertEquals(flds[0].getType(), SortField.REWRITEABLE);
 
     //Not thrilled about the fragility of string matching here, but...
     //the value sources get wrapped, so the out field is different than the input
@@ -137,7 +137,7 @@ public class QueryParsingTest extends SolrTestCaseJ4 {
     //Test literals in functions
     sort = QueryParsing.parseSort("strdist(foo_s1, \"junk\", jw) desc", req);
     flds = sort.getSort();
-    assertEquals(flds[0].getType(), SortField.CUSTOM);
+    assertEquals(flds[0].getType(), SortField.REWRITEABLE);
     //the value sources get wrapped, so the out field is different than the input
     assertEquals(flds[0].getField(), "strdist(str(foo_s1),literal(junk), dist=org.apache.lucene.search.spell.JaroWinklerDistance)");