From 921bd4791957746f137f5f9e21cb1b7d0d50a270 Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Thu, 28 Nov 2013 21:00:28 +0000 Subject: [PATCH] SOLR-5354: Distributed sort is broken with CUSTOM FieldType git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1546457 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 + .../apache/solr/schema/ICUCollationField.java | 20 ++ .../handler/component/QueryComponent.java | 53 ++-- .../solr/handler/component/ShardDoc.java | 169 ++++------- .../apache/solr/schema/CollationField.java | 20 ++ .../org/apache/solr/schema/FieldType.java | 16 + .../solr/schema/SortableDoubleField.java | 21 ++ .../solr/schema/SortableFloatField.java | 21 ++ .../apache/solr/schema/SortableIntField.java | 21 ++ .../apache/solr/schema/SortableLongField.java | 21 ++ .../java/org/apache/solr/schema/StrField.java | 23 ++ .../org/apache/solr/schema/TextField.java | 23 ++ .../collection1/conf/schema-custom-field.xml | 42 +++ .../conf/schema-distributed-missing-sort.xml | 83 +++++ .../solr/TestDistributedMissingSort.java | 287 ++++++++++++++++++ ...stributedQueryComponentCustomSortTest.java | 110 +++++++ .../solr/schema/SortableBinaryField.java | 103 +++++++ .../apache/solr/search/TestCustomSort.java | 126 ++++++++ .../solr/BaseDistributedSearchTestCase.java | 16 +- .../java/org/apache/solr/SolrTestCaseJ4.java | 26 ++ 20 files changed, 1057 insertions(+), 147 deletions(-) create mode 100644 solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml create mode 100644 solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java create mode 100644 solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java create mode 100644 solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java create mode 100644 solr/core/src/test/org/apache/solr/search/TestCustomSort.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 32b69c18ef0..6bbe418ecf3 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -159,6 +159,9 @@ Bug Fixes * SOLR-5494: CoreContainer#remove throws NPE rather than returning null when a SolrCore does not exist in core discovery mode. (Mark Miller) + +* SOLR-5354: Distributed sort is broken with CUSTOM FieldType. + (Steve Rowe, hossman, Jessica Cheng) Optimizations ---------------------- diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java index 30cba33d96e..7c33a3dd335 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java @@ -43,6 +43,7 @@ import org.apache.lucene.util.Version; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.util.Base64; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; @@ -300,4 +301,23 @@ public class ICUCollationField extends FieldType { return Collections.singletonList(createField(field, value, boost)); } } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + final BytesRef val = (BytesRef)value; + return Base64.byteArrayToBase64(val.bytes, val.offset, val.length); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + final String val = (String)value; + final byte[] bytes = Base64.base64ToByteArray(val); + return new BytesRef(bytes); + } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 0a020569a66..ca072e18dec 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -34,7 +34,6 @@ import org.apache.lucene.search.grouping.SearchGroup; import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; -import org.apache.lucene.util.UnicodeUtil; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -47,6 +46,7 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; @@ -449,7 +449,6 @@ public class QueryComponent extends SearchComponent { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; - final CharsRef spare = new CharsRef(); // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. @@ -458,7 +457,6 @@ public class QueryComponent extends SearchComponent Sort sort = searcher.weightSort(rb.getSortSpec().getSort()); SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort(); NamedList sortVals = new NamedList(); // order is important for the sort fields - Field field = new StringField("dummy", "", Field.Store.NO); // a dummy Field IndexReaderContext topReaderContext = searcher.getTopReaderContext(); List leaves = topReaderContext.leaves(); AtomicReaderContext currentLeaf = null; @@ -516,27 +514,7 @@ public class QueryComponent extends SearchComponent doc -= currentLeaf.docBase; // adjust for what segment this is in comparator.copy(0, doc); Object val = comparator.value(0); - - // Sortable float, double, int, long types all just use a string - // comparator. For these, we need to put the type into a readable - // format. One reason for this is that XML can't represent all - // string values (or even all unicode code points). - // indexedToReadable() should be a no-op and should - // thus be harmless anyway (for all current ways anyway) - if (val instanceof String) { - field.setStringValue((String)val); - val = ft.toObject(field); - } - - // Must do the same conversion when sorting by a - // String field in Lucene, which returns the terms - // data as BytesRef: - if (val instanceof BytesRef) { - UnicodeUtil.UTF8toUTF16((BytesRef)val, spare); - field.setStringValue(spare.toString()); - val = ft.toObject(field); - } - + if (null != ft) val = ft.marshalSortValue(val); vals[position] = val; } @@ -778,7 +756,8 @@ public class QueryComponent extends SearchComponent sortFields = new SortField[]{SortField.FIELD_SCORE}; } - SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField(); + IndexSchema schema = rb.req.getSchema(); + SchemaField uniqueKeyField = schema.getUniqueKeyField(); // id to shard mapping, to eliminate any accidental dups @@ -787,7 +766,7 @@ public class QueryComponent extends SearchComponent // Merge the docs via a priority queue so we don't have to sort *all* of the // documents... we only need to order the top (rows+start) ShardFieldSortedHitQueue queue; - queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount()); + queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount(), rb.req.getSearcher()); NamedList shardInfo = null; if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) { @@ -886,7 +865,7 @@ public class QueryComponent extends SearchComponent } } - shardDoc.sortFieldValues = sortFieldValues; + shardDoc.sortFieldValues = unmarshalSortValues(sortFieldValues, schema); queue.insertWithOverflow(shardDoc); } // end for-each-doc-in-response @@ -928,6 +907,26 @@ public class QueryComponent extends SearchComponent } } + private NamedList unmarshalSortValues(NamedList sortFieldValues, IndexSchema schema) { + NamedList unmarshalledSortValsPerField = new NamedList(); + for (int fieldNum = 0 ; fieldNum < sortFieldValues.size() ; ++fieldNum) { + String fieldName = sortFieldValues.getName(fieldNum); + SchemaField field = schema.getFieldOrNull(fieldName); + List sortVals = (List)sortFieldValues.getVal(fieldNum); + if (null == field) { + unmarshalledSortValsPerField.add(fieldName, sortVals); + } else { + FieldType fieldType = field.getType(); + List unmarshalledSortVals = new ArrayList(); + for (Object sortVal : sortVals) { + unmarshalledSortVals.add(fieldType.unmarshalSortValue(sortVal)); + } + unmarshalledSortValsPerField.add(fieldName, unmarshalledSortVals); + } + } + return unmarshalledSortValsPerField; + } + private void createRetrieveDocs(ResponseBuilder rb) { // TODO: in a system with nTiers > 2, we could be passed "ids" here diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java index 3ffb0a97ecc..603f262d925 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java @@ -16,18 +16,21 @@ */ package org.apache.solr.handler.component; -import org.apache.lucene.search.FieldComparatorSource; +import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SortField; import org.apache.lucene.util.PriorityQueue; +import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; -import org.apache.solr.search.MissingStringLastComparatorSource; +import org.apache.solr.search.SolrIndexSearcher; -import java.text.Collator; +import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Locale; + +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; public class ShardDoc extends FieldDoc { public String shard; @@ -101,7 +104,7 @@ public class ShardDoc extends FieldDoc { class ShardFieldSortedHitQueue extends PriorityQueue { /** Stores a comparator corresponding to each field being sorted by */ - protected Comparator[] comparators; + protected Comparator[] comparators; /** Stores the sort criteria being used. */ protected SortField[] fields; @@ -109,9 +112,10 @@ class ShardFieldSortedHitQueue extends PriorityQueue { /** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */ protected List fieldNames = new ArrayList(); - public ShardFieldSortedHitQueue(SortField[] fields, int size) { + public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) { super(size); final int n = fields.length; + //noinspection unchecked comparators = new Comparator[n]; this.fields = new SortField[n]; for (int i = 0; i < n; ++i) { @@ -123,8 +127,7 @@ class ShardFieldSortedHitQueue extends PriorityQueue { } String fieldname = fields[i].getField(); - comparators[i] = getCachedComparator(fieldname, fields[i] - .getType(), fields[i].getComparatorSource()); + comparators[i] = getCachedComparator(fields[i], searcher); if (fields[i].getType() == SortField.Type.STRING) { this.fields[i] = new SortField(fieldname, SortField.Type.STRING, @@ -169,47 +172,36 @@ class ShardFieldSortedHitQueue extends PriorityQueue { return c < 0; } - Comparator getCachedComparator(String fieldname, SortField.Type type, FieldComparatorSource factory) { - Comparator comparator = null; - switch (type) { - case SCORE: - comparator = comparatorScore(fieldname); - break; - case STRING: - comparator = comparatorNatural(fieldname); - break; - case CUSTOM: - if (factory instanceof MissingStringLastComparatorSource){ - comparator = comparatorMissingStringLast(fieldname); - } else { - // TODO: support other types such as random... is there a way to - // support generically? Perhaps just comparing Object - comparator = comparatorNatural(fieldname); - // throw new RuntimeException("Custom sort not supported factory is "+factory.getClass()); + Comparator getCachedComparator(SortField sortField, IndexSearcher searcher) { + SortField.Type type = sortField.getType(); + if (type == SortField.Type.SCORE) { + return comparatorScore(); + } else if (type == SortField.Type.REWRITEABLE) { + try { + sortField = sortField.rewrite(searcher); + } catch (IOException e) { + throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField, e); } - break; - case DOC: - // TODO: we can support this! - throw new RuntimeException("Doc sort not supported"); - default: - comparator = comparatorNatural(fieldname); - break; } - return comparator; + return comparatorFieldComparator(sortField); } - class ShardComparator implements Comparator { - String fieldName; - int fieldNum; - public ShardComparator(String fieldName) { - this.fieldName = fieldName; - this.fieldNum=0; + abstract class ShardComparator implements Comparator { + final SortField sortField; + final String fieldName; + final int fieldNum; + + public ShardComparator(SortField sortField) { + this.sortField = sortField; + this.fieldName = sortField.getField(); + int fieldNum = 0; for (int i=0; i { List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum); return lst.get(shardDoc.orderInShard); } - - @Override - public int compare(Object o1, Object o2) { - return 0; - } } - static Comparator comparatorScore(final String fieldName) { - return new Comparator() { + static Comparator comparatorScore() { + return new Comparator() { @Override - public final int compare(final Object o1, final Object o2) { - ShardDoc e1 = (ShardDoc) o1; - ShardDoc e2 = (ShardDoc) o2; - - final float f1 = e1.score; - final float f2 = e2.score; + public final int compare(final ShardDoc o1, final ShardDoc o2) { + final float f1 = o1.score; + final float f2 = o2.score; if (f1 < f2) return -1; if (f1 > f2) @@ -242,71 +226,24 @@ class ShardFieldSortedHitQueue extends PriorityQueue { }; } - // The lucene natural sort ordering corresponds to numeric - // and string natural sort orderings (ascending). Since - // the PriorityQueue keeps the biggest elements by default, - // we need to reverse the natural compare ordering so that the - // smallest elements are kept instead of the largest... hence - // the negative sign on the final compareTo(). - Comparator comparatorNatural(String fieldName) { - return new ShardComparator(fieldName) { + Comparator comparatorFieldComparator(SortField sortField) { + final FieldComparator fieldComparator; + try { + fieldComparator = sortField.getComparator(0, 0); + } catch (IOException e) { + throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField); + } + + return new ShardComparator(sortField) { + // Since the PriorityQueue keeps the biggest elements by default, + // we need to reverse the field compare ordering so that the + // smallest elements are kept instead of the largest... hence + // the negative sign. @Override - public final int compare(final Object o1, final Object o2) { - ShardDoc sd1 = (ShardDoc) o1; - ShardDoc sd2 = (ShardDoc) o2; - Comparable v1 = (Comparable)sortVal(sd1); - Comparable v2 = (Comparable)sortVal(sd2); - if (v1==v2) - return 0; - if (v1==null) - return 1; - if(v2==null) - return -1; - return -v1.compareTo(v2); + public int compare(final ShardDoc o1, final ShardDoc o2) { + //noinspection unchecked + return -fieldComparator.compareValues(sortVal(o1), sortVal(o2)); } }; } - - - Comparator comparatorStringLocale(final String fieldName, - Locale locale) { - final Collator collator = Collator.getInstance(locale); - return new ShardComparator(fieldName) { - @Override - public final int compare(final Object o1, final Object o2) { - ShardDoc sd1 = (ShardDoc) o1; - ShardDoc sd2 = (ShardDoc) o2; - Comparable v1 = (Comparable)sortVal(sd1); - Comparable v2 = (Comparable)sortVal(sd2); - if (v1==v2) - return 0; - if (v1==null) - return 1; - if(v2==null) - return -1; - return -collator.compare(v1,v2); - } - }; - } - - - Comparator comparatorMissingStringLast(final String fieldName) { - return new ShardComparator(fieldName) { - @Override - public final int compare(final Object o1, final Object o2) { - ShardDoc sd1 = (ShardDoc) o1; - ShardDoc sd2 = (ShardDoc) o2; - Comparable v1 = (Comparable)sortVal(sd1); - Comparable v2 = (Comparable)sortVal(sd2); - if (v1==v2) - return 0; - if (v1==null) - return -1; - if(v2==null) - return 1; - return -v1.compareTo(v2); - } - }; - } - } diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.java index cfea5f348d6..5b46159bae4 100644 --- a/solr/core/src/java/org/apache/solr/schema/CollationField.java +++ b/solr/core/src/java/org/apache/solr/schema/CollationField.java @@ -47,6 +47,7 @@ import org.apache.lucene.util.Version; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.util.Base64; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; @@ -275,4 +276,23 @@ public class CollationField extends FieldType { return Collections.singletonList(createField(field, value, boost)); } } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + final BytesRef val = (BytesRef)value; + return Base64.byteArrayToBase64(val.bytes, val.offset, val.length); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + final String val = (String)value; + final byte[] bytes = Base64.base64ToByteArray(val); + return new BytesRef(bytes); + } } diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 5d03d383060..8d2a1d3aded 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -932,4 +932,20 @@ public abstract class FieldType extends FieldProperties { } return analyzerProps; } + + /** + * Convert a value used by the FieldComparator for this FieldType's SortField + * into a marshalable value for distributed sorting. + */ + public Object marshalSortValue(Object value) { + return value; + } + + /** + * Convert a value marshaled via {@link #marshalSortValue} back + * into a value usable by the FieldComparator for this FieldType's SortField + */ + public Object unmarshalSortValue(Object value) { + return value; + } } diff --git a/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java b/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java index 2f4b0a65dda..382dfd430d4 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java @@ -100,6 +100,27 @@ public class SortableDoubleField extends PrimitiveFieldType implements DoubleVal String sval = f.stringValue(); writer.writeDouble(name, NumberUtils.SortableStr2double(sval)); } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + CharsRef chars = new CharsRef(); + UnicodeUtil.UTF8toUTF16((BytesRef)value, chars); + return NumberUtils.SortableStr2double(chars.toString()); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + String sortableString = NumberUtils.double2sortableStr(value.toString()); + BytesRef bytes = new BytesRef(); + UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes); + return bytes; + } } class SortableDoubleFieldSource extends FieldCacheSource { diff --git a/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java b/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java index e66e25563e7..aa7a075c867 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java @@ -101,6 +101,27 @@ public class SortableFloatField extends PrimitiveFieldType implements FloatValue String sval = f.stringValue(); writer.writeFloat(name, NumberUtils.SortableStr2float(sval)); } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + CharsRef chars = new CharsRef(); + UnicodeUtil.UTF8toUTF16((BytesRef)value, chars); + return NumberUtils.SortableStr2float(chars.toString()); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + String sortableString = NumberUtils.float2sortableStr(value.toString()); + BytesRef bytes = new BytesRef(); + UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes); + return bytes; + } } diff --git a/solr/core/src/java/org/apache/solr/schema/SortableIntField.java b/solr/core/src/java/org/apache/solr/schema/SortableIntField.java index 955857370f9..97cbfe2b134 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableIntField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableIntField.java @@ -104,6 +104,27 @@ public class SortableIntField extends PrimitiveFieldType implements IntValueFiel String sval = f.stringValue(); writer.writeInt(name, NumberUtils.SortableStr2int(sval,0,sval.length())); } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + CharsRef chars = new CharsRef(); + UnicodeUtil.UTF8toUTF16((BytesRef)value, chars); + return NumberUtils.SortableStr2int(chars.toString()); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + String sortableString = NumberUtils.int2sortableStr(value.toString()); + BytesRef bytes = new BytesRef(); + UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes); + return bytes; + } } diff --git a/solr/core/src/java/org/apache/solr/schema/SortableLongField.java b/solr/core/src/java/org/apache/solr/schema/SortableLongField.java index 0e61eef6f91..3c48af14bad 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableLongField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableLongField.java @@ -100,6 +100,27 @@ public class SortableLongField extends PrimitiveFieldType { String sval = f.stringValue(); writer.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length())); } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + CharsRef chars = new CharsRef(); + UnicodeUtil.UTF8toUTF16((BytesRef)value, chars); + return NumberUtils.SortableStr2long(chars.toString()); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + String sortableString = NumberUtils.long2sortableStr(value.toString()); + BytesRef bytes = new BytesRef(); + UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes); + return bytes; + } } diff --git a/solr/core/src/java/org/apache/solr/schema/StrField.java b/solr/core/src/java/org/apache/solr/schema/StrField.java index 2c9600c67de..15060b9b74d 100644 --- a/solr/core/src/java/org/apache/solr/schema/StrField.java +++ b/solr/core/src/java/org/apache/solr/schema/StrField.java @@ -29,6 +29,8 @@ import org.apache.lucene.index.StorableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.UnicodeUtil; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; @@ -81,6 +83,27 @@ public class StrField extends PrimitiveFieldType { @Override public void checkSchemaField(SchemaField field) { } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + CharsRef spare = new CharsRef(); + UnicodeUtil.UTF8toUTF16((BytesRef)value, spare); + return spare.toString(); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + BytesRef spare = new BytesRef(); + String stringVal = (String)value; + UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare); + return spare; + } } diff --git a/solr/core/src/java/org/apache/solr/schema/TextField.java b/solr/core/src/java/org/apache/solr/schema/TextField.java index b7fd860f11a..f0741f51445 100644 --- a/solr/core/src/java/org/apache/solr/schema/TextField.java +++ b/solr/core/src/java/org/apache/solr/schema/TextField.java @@ -23,7 +23,9 @@ import org.apache.lucene.index.StorableField; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.QueryBuilder; +import org.apache.lucene.util.UnicodeUtil; import org.apache.solr.common.SolrException; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; @@ -165,4 +167,25 @@ public class TextField extends FieldType { public boolean isExplicitMultiTermAnalyzer() { return isExplicitMultiTermAnalyzer; } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + CharsRef spare = new CharsRef(); + UnicodeUtil.UTF8toUTF16((BytesRef)value, spare); + return spare.toString(); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + BytesRef spare = new BytesRef(); + String stringVal = (String)value; + UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare); + return spare; + } } diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml new file mode 100644 index 00000000000..de206bce0bb --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + text + id + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml new file mode 100644 index 00000000000..c78c11c3da1 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml @@ -0,0 +1,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java b/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java new file mode 100644 index 00000000000..6f7956d9d22 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr; + +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.solr.client.solrj.response.QueryResponse; + +/** + * Tests sortMissingFirst and sortMissingLast in distributed sort + */ +@Slow +public class TestDistributedMissingSort extends BaseDistributedSearchTestCase { + + public TestDistributedMissingSort() { + schemaString = "schema-distributed-missing-sort.xml"; + } + + String sint1_ml = "one_si_ml"; // SortableIntField, sortMissingLast=true, multiValued=false + String sint1_mf = "two_si_mf"; // SortableIntField, sortMissingFirst=true, multiValued=false + String long1_ml = "three_l1_ml"; // TrieLongField, sortMissingLast=true, multiValued=false + String long1_mf = "four_l1_mf"; // TrieLongField, sortMissingFirst=true, multiValued=false + String string1_ml = "five_s1_ml"; // StringField, sortMissingLast=true, multiValued=false + String string1_mf = "six_s1_mf"; // StringField, sortMissingFirst=true, multiValued=false + + @Override + public void doTest() throws Exception { + index(); + testSortMissingLast(); + testSortMissingFirst(); + } + + private void index() throws Exception { + del("*:*"); + indexr(id,1, sint1_ml, 100, sint1_mf, 100, long1_ml, 100, long1_mf, 100, + "foo_f", 1.414f, "foo_b", "true", "foo_d", 1.414d, + string1_ml, "DE", string1_mf, "DE"); + indexr(id,2, sint1_ml, 50, sint1_mf, 50, long1_ml, 50, long1_mf, 50, + string1_ml, "ABC", string1_mf, "ABC"); + indexr(id,3, sint1_ml, 2, sint1_mf, 2, long1_ml, 2, long1_mf, 2, + string1_ml, "HIJK", string1_mf, "HIJK"); + indexr(id,4, sint1_ml, -100, sint1_mf, -100, long1_ml, -101, long1_mf, -101, + string1_ml, "L M", string1_mf, "L M"); + indexr(id,5, sint1_ml, 500, sint1_mf, 500, long1_ml, 500, long1_mf, 500, + string1_ml, "YB", string1_mf, "YB"); + indexr(id,6, sint1_ml, -600, sint1_mf, -600, long1_ml, -600, long1_mf, -600, + string1_ml, "WX", string1_mf, "WX"); + indexr(id,7, sint1_ml, 123, sint1_mf, 123, long1_ml, 123, long1_mf, 123, + string1_ml, "N", string1_mf, "N"); + indexr(id,8, sint1_ml, 876, sint1_mf, 876, long1_ml, 876, long1_mf, 876, + string1_ml, "QRS", string1_mf, "QRS"); + indexr(id,9, sint1_ml, 7, sint1_mf, 7, long1_ml, 7, long1_mf, 7, + string1_ml, "P", string1_mf, "P"); + + commit(); // try to ensure there's more than one segment + + indexr(id,10, sint1_ml, 4321, sint1_mf, 4321, long1_ml, 4321, long1_mf, 4321, + string1_ml, "O", string1_mf, "O"); + indexr(id,11, sint1_ml, -987, sint1_mf, -987, long1_ml, -987, long1_mf, -987, + string1_ml, "YA", string1_mf, "YA"); + indexr(id,12, sint1_ml, 379, sint1_mf, 379, long1_ml, 379, long1_mf, 379, + string1_ml, "TUV", string1_mf, "TUV"); + indexr(id,13, sint1_ml, 232, sint1_mf, 232, long1_ml, 232, long1_mf, 232, + string1_ml, "F G", string1_mf, "F G"); + + indexr(id, 14, "SubjectTerms_mfacet", new String[] {"mathematical models", "mathematical analysis"}); + indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); + indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); + String[] vals = new String[100]; + for (int i=0; i<100; i++) { + vals[i] = "test " + i; + } + indexr(id, 17, "SubjectTerms_mfacet", vals); + + for (int i=100; i<150; i++) { + indexr(id, i); + } + + commit(); + + handle.clear(); + handle.put("QTime", SKIPVAL); + handle.put("timestamp", SKIPVAL); + handle.put("_version_", SKIPVAL); // not a cloud test, but may use updateLog + } + + private void testSortMissingLast() throws Exception { + // id field values: 1 2 3 4 5 6 7 8 9 10 11 12 13 + // sint1_ml field values: 100 50 2 -100 500 -600 123 876 7 4321 -987 379 232 + // sint1_ml asc sort pos: 7 6 4 3 11 2 8 12 5 13 1 10 9 + // sint1_ml desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 + + QueryResponse rsp = query("q","*:*", "sort", sint1_ml + " desc", "rows", "13"); + assertFieldValues(rsp.getResults(), id, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); + + rsp = query("q","*:*", "sort", sint1_ml + " asc", "rows", "13"); + assertFieldValues(rsp.getResults(), id, 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); + + rsp = query("q","*:*", "sort", sint1_ml + " desc," + id + " asc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11, + // missing field sint1_ml="a_si", ascending id sort + 14, 15, 16, 17, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149); + + rsp = query("q","*:*", "sort", sint1_ml + " asc," + id + " desc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10, + // missing field sint1_ml="a_si", descending id sort + 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, + 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, + 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, + 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, + 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, + 17, 16, 15, 14); + + // id field values: 1 2 3 4 5 6 7 8 9 10 11 12 13 + // long1_ml field values: 100 50 2 -100 500 -600 123 876 7 4321 -987 379 232 + // long1_ml asc sort pos: 7 6 4 3 11 2 8 12 5 13 1 10 9 + // long1_ml desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 + + rsp = query("q","*:*", "sort", long1_ml + " desc", "rows", "13"); + assertFieldValues(rsp.getResults(), id, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); + + rsp = query("q","*:*", "sort", long1_ml + " asc", "rows", "13"); + assertFieldValues(rsp.getResults(), id, 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); + + rsp = query("q","*:*", "sort", long1_ml + " desc," + id + " asc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11, + // missing field sint1_ml="a_si", ascending id sort + 14, 15, 16, 17, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149); + + rsp = query("q","*:*", "sort", long1_ml + " asc," + id + " desc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10, + // missing field sint1_ml="a_si", descending id sort + 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, + 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, + 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, + 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, + 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, + 17, 16, 15, 14); + + + // id field values: 1 2 3 4 5 6 7 8 9 10 11 12 13 + // string1_ml field values: DE ABC HIJK L M YB WX N QRS P O YA TUV F G + // string1_ml asc sort pos: 2 1 4 5 13 11 6 9 8 7 12 10 3 + // string1_ml desc sort pos: 12 13 10 9 1 3 8 5 6 7 2 4 11 + + rsp = query("q","*:*", "sort", string1_ml + " desc", "rows", "13"); + assertFieldValues(rsp.getResults(), id, 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2); + + rsp = query("q","*:*", "sort", string1_ml + " asc", "rows", "13"); + assertFieldValues(rsp.getResults(), id, 2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5); + + rsp = query("q","*:*", "sort", string1_ml + " desc," + id + " asc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2, + // missing field string1_ml="a_s1", ascending id sort + 14, 15, 16, 17, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149); + + rsp = query("q","*:*", "sort", string1_ml + " asc," + id + " desc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + 2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5, + // missing field string1_ml="a_s1", descending id sort + 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, + 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, + 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, + 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, + 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, + 17, 16, 15, 14); + } + + private void testSortMissingFirst() throws Exception { + // id field values: 1 2 3 4 5 6 7 8 9 10 11 12 13 + // sint1_mf field values: 100 50 2 -100 500 -600 123 876 7 4321 -987 379 232 + // sint1_mf asc sort pos: 7 6 4 3 11 2 8 12 5 13 1 10 9 + // sint1_mf desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 + + QueryResponse rsp = query("q","*:*", "sort", sint1_mf + " desc," + id + " asc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + // missing field sint1_mf="a_si_mf", ascending id sort + 14, 15, 16, 17, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); + + rsp = query("q","*:*", "sort", sint1_mf + " asc," + id + " desc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + // missing field sint1_mf="a_si_mf", descending id sort + 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, + 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, + 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, + 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, + 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, + 17, 16, 15, 14, + 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); + + + // id field values: 1 2 3 4 5 6 7 8 9 10 11 12 13 + // long1_mf field values: 100 50 2 -100 500 -600 123 876 7 4321 -987 379 232 + // long1_mf asc sort pos: 7 6 4 3 11 2 8 12 5 13 1 10 9 + // long1_mf desc sort pos: 7 8 10 11 3 12 6 2 9 1 13 4 5 + + rsp = query("q","*:*", "sort", long1_mf + " desc," + id + " asc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + // missing field sint1_mf="a_si_mf", ascending id sort + 14, 15, 16, 17, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11); + + rsp = query("q","*:*", "sort", long1_mf + " asc," + id + " desc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + // missing field sint1_mf="a_si_mf", descending id sort + 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, + 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, + 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, + 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, + 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, + 17, 16, 15, 14, + 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10); + + + // id field values: 1 2 3 4 5 6 7 8 9 10 11 12 13 + // string1_mf field values: DE ABC HIJK L M YB WX N QRS P O YA TUV F G + // string1_mf asc sort pos: 2 1 4 5 13 11 6 9 8 7 12 10 3 + // string1_mf desc sort pos: 12 13 10 9 1 3 8 5 6 7 2 4 11 + + rsp = query("q","*:*", "sort", string1_mf + " desc," + id + " asc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + // missing field string1_mf="a_s1_mf", ascending id sort + 14, 15, 16, 17, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2); + + rsp = query("q","*:*", "sort", string1_mf + " asc," + id + " desc", "rows", "200"); + assertFieldValues(rsp.getResults(), id, + // missing field string1_mf="a_s1_mf", descending id sort + 149, 148, 147, 146, 145, 144, 143, 142, 141, 140, + 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, + 129, 128, 127, 126, 125, 124, 123, 122, 121, 120, + 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, + 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, + 17, 16, 15, 14, + 2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java new file mode 100644 index 00000000000..8a4b9735e0b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java @@ -0,0 +1,110 @@ +package org.apache.solr.handler.component; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.solr.BaseDistributedSearchTestCase; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.junit.BeforeClass; + +import java.nio.ByteBuffer; + +/** + * Test for QueryComponent's distributed querying + * + * @see org.apache.solr.handler.component.QueryComponent + */ +public class DistributedQueryComponentCustomSortTest extends BaseDistributedSearchTestCase { + + public DistributedQueryComponentCustomSortTest() { + fixShardCount = true; + shardCount = 3; + stress = 0; + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + initCore("solrconfig.xml", "schema-custom-field.xml"); + } + + @Override + public void doTest() throws Exception { + del("*:*"); + + index(id, "1", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x12, 0x62, 0x15 })); // 2 + index(id, "2", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x16 })); // 5 + index(id, "3", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x32, 0x58 })); // 8 + index(id, "4", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 })); // 4 + index(id, "5", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x35, 0x10, 0x00 })); // 9 + index(id, "6", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x1a, 0x2b, 0x3c, 0x00, 0x00, 0x03 })); // 3 + index(id, "7", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x00, 0x3c, 0x73 })); // 1 + index(id, "8", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x59, 0x2d, 0x4d })); // 11 + index(id, "9", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x39, 0x79, 0x7a })); // 10 + index(id, "10", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c })); // 6 + index(id, "11", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0xff, (byte)0xaf, (byte)0x9c })); // 13 + index(id, "12", "text", "d", "payload", ByteBuffer.wrap(new byte[] { 0x34, (byte)0xdd, 0x4d })); // 7 + index(id, "13", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0x80, 0x11, 0x33 })); // 12 + commit(); + + handle.put("QTime", SKIPVAL); + + QueryResponse rsp; + rsp = query("q", "*:*", "fl", "id", "sort", "payload asc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11); + rsp = query("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12, 10, 2, 4, 6, 1, 7); + + rsp = query("q", "text:a", "fl", "id", "sort", "payload asc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 1, 3, 5, 9); + rsp = query("q", "text:a", "fl", "id", "sort", "payload desc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 9, 5, 3, 1); + + rsp = query("q", "text:b", "fl", "id", "sort", "payload asc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 4, 2, 10); + rsp = query("q", "text:b", "fl", "id", "sort", "payload desc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 10, 2, 4); + + rsp = query("q", "text:c", "fl", "id", "sort", "payload asc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 7, 6, 8); + rsp = query("q", "text:c", "fl", "id", "sort", "payload desc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 8, 6, 7); + + rsp = query("q", "text:d", "fl", "id", "sort", "payload asc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 12, 13, 11); + rsp = query("q", "text:d", "fl", "id", "sort", "payload desc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 11, 13, 12); + + + // Add two more docs with same payload as in doc #4 + index(id, "14", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 })); + index(id, "15", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 })); + + // Add three more docs with same payload as in doc #10 + index(id, "16", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c })); + index(id, "17", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c })); + index(id, "18", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c })); + + commit(); + + rsp = query("q", "*:*", "fl", "id", "sort", "payload asc, id desc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 7, 1, 6, 15,14,4, 2, 18,17,16,10, 12, 3, 5, 9, 8, 13, 11); + rsp = query("q", "*:*", "fl", "id", "sort", "payload desc, id asc", "rows", "20"); + assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12, 10,16,17,18, 2, 4,14,15, 6, 1, 7); + } +} diff --git a/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java b/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java new file mode 100644 index 00000000000..1fa86d6eb36 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java @@ -0,0 +1,103 @@ +package org.apache.solr.schema; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.StorableField; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldComparatorSource; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.util.Base64; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Custom field representing a {@link BinaryField} that's sortable. + */ +public class SortableBinaryField extends BinaryField { + + @Override + public void checkSchemaField(final SchemaField field) { + if (field.hasDocValues() && !field.multiValued() && !(field.isRequired() || field.getDefaultValue() != null)) { + throw new IllegalStateException( + "Field " + this + " has single-valued doc values enabled, but has no default value and is not required"); + } + } + + @Override + public List createFields(SchemaField field, Object value, float boost) { + if (field.hasDocValues()) { + List fields = new ArrayList(); + StorableField storedField = createField(field, value, boost); + fields.add(storedField); + ByteBuffer byteBuffer = toObject(storedField); + BytesRef bytes = new BytesRef + (byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining()); + if (field.multiValued()) { + fields.add(new SortedSetDocValuesField(field.getName(), bytes)); + } else { + fields.add(new SortedDocValuesField(field.getName(), bytes)); + } + return fields; + } else { + return Collections.singletonList(createField(field, value, boost)); + } + } + + @Override + public SortField getSortField(final SchemaField field, final boolean reverse) { + field.checkSortability(); + return new BinarySortField(field.getName(), reverse); + } + + private static class BinarySortField extends SortField { + public BinarySortField(final String field, final boolean reverse) { + super(field, new FieldComparatorSource() { + @Override + public FieldComparator.TermOrdValComparator newComparator + (final String fieldname, final int numHits, final int sortPos, final boolean reversed) throws IOException { + return new FieldComparator.TermOrdValComparator(numHits, fieldname); + }}, reverse); + } + } + + @Override + public Object marshalSortValue(Object value) { + if (null == value) { + return null; + } + final BytesRef val = (BytesRef)value; + return Base64.byteArrayToBase64(val.bytes, val.offset, val.length); + } + + @Override + public Object unmarshalSortValue(Object value) { + if (null == value) { + return null; + } + final String val = (String)value; + final byte[] bytes = Base64.base64ToByteArray(val); + return new BytesRef(bytes); + } +} diff --git a/solr/core/src/test/org/apache/solr/search/TestCustomSort.java b/solr/core/src/test/org/apache/solr/search/TestCustomSort.java new file mode 100644 index 00000000000..d2afe4d3f6c --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestCustomSort.java @@ -0,0 +1,126 @@ +package org.apache.solr.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.solr.SolrTestCaseJ4; +import org.junit.BeforeClass; + +import java.nio.ByteBuffer; + + +/** + * Test SortField.CUSTOM sorts + */ +public class TestCustomSort extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema-custom-field.xml"); + } + + public void testSortableBinary() throws Exception { + clearIndex(); + assertU(adoc(sdoc("id", "1", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x12, 0x62, 0x15 })))); // 2 + assertU(adoc(sdoc("id", "2", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x16 })))); // 5 + assertU(adoc(sdoc("id", "3", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x32, 0x58 })))); // 8 + assertU(adoc(sdoc("id", "4", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 })))); // 4 + assertU(adoc(sdoc("id", "5", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x35, 0x10, 0x00 })))); // 9 + assertU(adoc(sdoc("id", "6", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x1a, 0x2b, 0x3c, 0x00, 0x00, 0x03 })))); // 3 + assertU(adoc(sdoc("id", "7", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x00, 0x3c, 0x73 })))); // 1 + assertU(adoc(sdoc("id", "8", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x59, 0x2d, 0x4d })))); // 11 + assertU(adoc(sdoc("id", "9", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x39, 0x79, 0x7a })))); // 10 + assertU(adoc(sdoc("id", "10", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c })))); // 6 + assertU(adoc(sdoc("id", "11", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0xff, (byte)0xaf, (byte)0x9c })))); // 13 + assertU(adoc(sdoc("id", "12", "text", "d", "payload", ByteBuffer.wrap(new byte[] { 0x34, (byte)0xdd, 0x4d })))); // 7 + assertU(adoc(sdoc("id", "13", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0x80, 0x11, 0x33 })))); // 12 + assertU(commit()); + + assertQ(req("q", "*:*", "fl", "id", "sort", "payload asc", "rows", "20") + , "//result[@numFound='13']" // + , "//result/doc[int='7' and position()=1]" // 7 00 3c 73 + , "//result/doc[int='1' and position()=2]" // 1 12 62 15 + , "//result/doc[int='6' and position()=3]" // 6 1a 2b 3c 00 00 03 + , "//result/doc[int='4' and position()=4]" // 4 25 21 15 + , "//result/doc[int='2' and position()=5]" // 2 25 21 16 + , "//result/doc[int='10' and position()=6]" // 10 31 39 7c + , "//result/doc[int='12' and position()=7]" // 12 34 dd 4d + , "//result/doc[int='3' and position()=8]" // 3 35 32 58 + , "//result/doc[int='5' and position()=9]" // 5 35 35 10 00 + , "//result/doc[int='9' and position()=10]" // 9 39 79 7a + , "//result/doc[int='8' and position()=11]" // 8 59 2d 4d + , "//result/doc[int='13' and position()=12]" // 13 80 11 33 + , "//result/doc[int='11' and position()=13]"); // 11 ff af 9c + assertQ(req("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20") + , "//result[@numFound='13']" // + , "//result/doc[int='11' and position()=1]" // 11 ff af 9c + , "//result/doc[int='13' and position()=2]" // 13 80 11 33 + , "//result/doc[int='8' and position()=3]" // 8 59 2d 4d + , "//result/doc[int='9' and position()=4]" // 9 39 79 7a + , "//result/doc[int='5' and position()=5]" // 5 35 35 10 00 + , "//result/doc[int='3' and position()=6]" // 3 35 32 58 + , "//result/doc[int='12' and position()=7]" // 12 34 dd 4d + , "//result/doc[int='10' and position()=8]" // 10 31 39 7c + , "//result/doc[int='2' and position()=9]" // 2 25 21 16 + , "//result/doc[int='4' and position()=10]" // 4 25 21 15 + , "//result/doc[int='6' and position()=11]" // 6 1a 2b 3c 00 00 03 + , "//result/doc[int='1' and position()=12]" // 1 12 62 15 + , "//result/doc[int='7' and position()=13]"); // 7 00 3c 73 + assertQ(req("q", "text:a", "fl", "id", "sort", "payload asc", "rows", "20") + , "//result[@numFound='4']" // + , "//result/doc[int='1' and position()=1]" // 1 12 62 15 + , "//result/doc[int='3' and position()=2]" // 3 35 32 58 + , "//result/doc[int='5' and position()=3]" // 5 35 35 10 00 + , "//result/doc[int='9' and position()=4]"); // 9 39 79 7a + assertQ(req("q", "text:a", "fl", "id", "sort", "payload desc", "rows", "20") + , "//result[@numFound='4']" // + , "//result/doc[int='9' and position()=1]" // 9 39 79 7a + , "//result/doc[int='5' and position()=2]" // 5 35 35 10 00 + , "//result/doc[int='3' and position()=3]" // 3 35 32 58 + , "//result/doc[int='1' and position()=4]"); // 1 12 62 15 + assertQ(req("q", "text:b", "fl", "id", "sort", "payload asc", "rows", "20") + , "//result[@numFound='3']" // + , "//result/doc[int='4' and position()=1]" // 4 25 21 15 + , "//result/doc[int='2' and position()=2]" // 2 25 21 16 + , "//result/doc[int='10' and position()=3]"); // 10 31 39 7c + assertQ(req("q", "text:b", "fl", "id", "sort", "payload desc", "rows", "20") + , "//result[@numFound='3']" // + , "//result/doc[int='10' and position()=1]" // 10 31 39 7c + , "//result/doc[int='2' and position()=2]" // 2 25 21 16 + , "//result/doc[int='4' and position()=3]"); // 4 25 21 15 + assertQ(req("q", "text:c", "fl", "id", "sort", "payload asc", "rows", "20") + , "//result[@numFound='3']" // + , "//result/doc[int='7' and position()=1]" // 7 00 3c 73 + , "//result/doc[int='6' and position()=2]" // 6 1a 2b 3c 00 00 03 + , "//result/doc[int='8' and position()=3]"); // 8 59 2d 4d + assertQ(req("q", "text:c", "fl", "id", "sort", "payload desc", "rows", "20") + , "//result[@numFound='3']" // + , "//result/doc[int='8' and position()=1]" // 8 59 2d 4d + , "//result/doc[int='6' and position()=2]" // 6 1a 2b 3c 00 00 03 + , "//result/doc[int='7' and position()=3]"); // 7 00 3c 73 + assertQ(req("q", "text:d", "fl", "id", "sort", "payload asc", "rows", "20") + , "//result[@numFound='3']" // + , "//result/doc[int='12' and position()=1]" // 12 34 dd 4d + , "//result/doc[int='13' and position()=2]" // 13 80 11 33 + , "//result/doc[int='11' and position()=3]"); // 11 ff af 9c + assertQ(req("q", "text:d", "fl", "id", "sort", "payload desc", "rows", "20") + , "//result[@numFound='3']" // + , "//result/doc[int='11' and position()=1]" // 11 ff af 9c + , "//result/doc[int='13' and position()=2]" // 13 80 11 33 + , "//result/doc[int='12' and position()=3]"); // 12 34 dd 4d + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java index 1a8ebf43d53..877bed1f4f2 100644 --- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java @@ -507,11 +507,18 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 { return rsp; } - protected void query(Object... q) throws Exception { - query(true, q); + /** + * Sets distributed params. + * Returns the QueryResponse from {@link #queryServer}, + */ + protected QueryResponse query(Object... q) throws Exception { + return query(true, q); } - - protected void query(boolean setDistribParams, Object[] q) throws Exception { + + /** + * Returns the QueryResponse from {@link #queryServer} + */ + protected QueryResponse query(boolean setDistribParams, Object[] q) throws Exception { final ModifiableSolrParams params = new ModifiableSolrParams(); @@ -558,6 +565,7 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 { thread.join(); } } + return rsp; } public QueryResponse queryAndCompare(SolrParams params, SolrServer... servers) throws SolrServerException { diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 6c8faecf2c2..61f2d1be823 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -47,6 +47,8 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.QuickPatchThreadsFilter; import org.apache.lucene.util._TestUtil; import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; @@ -1627,6 +1629,30 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { throw new RuntimeException("XPath is invalid", e2); } } + + /** + * Fails if the number of documents in the given SolrDocumentList differs + * from the given number of expected values, or if any of the values in the + * given field don't match the expected values in the same order. + */ + public static void assertFieldValues(SolrDocumentList documents, String fieldName, Object... expectedValues) { + if (documents.size() != expectedValues.length) { + fail("Number of documents (" + documents.size() + + ") is different from number of expected values (" + expectedValues.length); + } + for (int docNum = 1 ; docNum <= documents.size() ; ++docNum) { + SolrDocument doc = documents.get(docNum - 1); + Object expected = expectedValues[docNum - 1]; + Object actual = doc.get(fieldName); + if (null != expected && null != actual) { + if ( ! expected.equals(actual)) { + fail( "Unexpected " + fieldName + " field value in document #" + docNum + + ": expected=[" + expected + "], actual=[" + actual + "]"); + } + } + } + } + public static void copyMinConf(File dstRoot) throws IOException { copyMinConf(dstRoot, null); }