honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add max_children limit to nested sort (#33587) 

Add an option to `nested` sort to limit the number of children to visit when picking the sort value
of the root document. 

Closes #33592
This commit is contained in:
eray 2018-10-05 13:02:47 +03:00 committed by Jim Ferenczi
parent 732ab06ee4
commit daf88335d7
17 changed files with 275 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/reference/search/request/sort.asciidoc
View File

@ -127,6 +127,9 @@ field support has a `nested` sort option with the following properties:
should match with in order for its field values to be taken into account
by sorting. Common case is to repeat the query / filter inside the
nested filter or query. By default no `nested_filter` is active.
`max_children`::
The maximum number of children to consider per root document
when picking the sort value. Defaults to unlimited.
`nested`::
Same as top-level `nested` but applies to another nested path within the
current nested object.

7
server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
View File

@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.search.sort.NestedSortBuilder;
import java.io.IOException;
@ -129,10 +130,12 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
private final BitSetProducer rootFilter;
private final Query innerQuery;
private final NestedSortBuilder nestedSort;
public Nested(BitSetProducer rootFilter, Query innerQuery) {
public Nested(BitSetProducer rootFilter, Query innerQuery, NestedSortBuilder nestedSort) {
this.rootFilter = rootFilter;
this.innerQuery = innerQuery;
this.nestedSort = nestedSort;
}
public Query getInnerQuery() {
@ -143,6 +146,8 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
return rootFilter;
}
public NestedSortBuilder getNestedSort() { return nestedSort; }
/**
* Get a {@link BitDocIdSet} that matches the root documents.
*/

7
server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
View File

@ -91,7 +91,9 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat
} else {
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, rootDocs, innerDocs);
final int maxChildren = nested.getNestedSort() != null ?
nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
selectedValues = sortMode.select(values, rootDocs, innerDocs, maxChildren);
}
if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) {
return selectedValues;
@ -119,7 +121,8 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat
} else {
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, missingBytes, rootDocs, innerDocs, context.reader().maxDoc());
final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
selectedValues = sortMode.select(values, missingBytes, rootDocs, innerDocs, context.reader().maxDoc(), maxChildren);
}
return selectedValues;
}

3
server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
View File

@ -76,7 +76,8 @@ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparato
} else {
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc(), maxChildren);
}
return selectedValues.getRawDoubleValues();
}

3
server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
View File

@ -68,7 +68,8 @@ public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparator
} else {
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc(), maxChildren);
}
return selectedValues.getRawFloatValues();
}

3
server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
View File

@ -67,7 +67,8 @@ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorS
} else {
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc(), maxChildren);
}
return selectedValues;
}

156
server/src/main/java/org/elasticsearch/search/MultiValueMode.java
View File

@ -48,7 +48,6 @@ import java.util.Locale;
* Defines what values to pick in the case a document contains multiple values for a particular field.
*/
public enum MultiValueMode implements Writeable {
/**
* Pick the sum of all the values.
*/
@ -64,16 +63,21 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
int totalCount = 0;
long totalValue = 0;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
final int count = values.docValueCount();
for (int index = 0; index < count; ++index) {
if (++count > maxChildren) {
break;
}
final int docCount = values.docValueCount();
for (int index = 0; index < docCount; ++index) {
totalValue += values.nextValue();
}
totalCount += count;
totalCount += docCount;
}
}
return totalCount > 0 ? totalValue : missingValue;
@ -90,18 +94,23 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
int totalCount = 0;
double totalValue = 0;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
final int count = values.docValueCount();
for (int index = 0; index < count; ++index) {
if (++count > maxChildren) {
break;
}
final int docCount = values.docValueCount();
for (int index = 0; index < docCount; ++index) {
totalValue += values.nextValue();
}
totalCount += count;
totalCount += docCount;
}
}
return totalCount > 0 ? totalValue : missingValue;
}
},
@ -121,16 +130,20 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
int totalCount = 0;
long totalValue = 0;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
final int count = values.docValueCount();
for (int index = 0; index < count; ++index) {
if (++count > maxChildren) {
break;
}
final int docCount = values.docValueCount();
for (int index = 0; index < docCount; ++index) {
totalValue += values.nextValue();
}
totalCount += count;
totalCount += docCount;
}
}
if (totalCount < 1) {
@ -150,16 +163,20 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
int totalCount = 0;
double totalValue = 0;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
final int count = values.docValueCount();
for (int index = 0; index < count; ++index) {
if (++count > maxChildren) {
break;
}
final int docCount = values.docValueCount();
for (int index = 0; index < docCount; ++index) {
totalValue += values.nextValue();
}
totalCount += count;
totalCount += docCount;
}
}
if (totalCount < 1) {
@ -210,11 +227,15 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
boolean hasValue = false;
long minValue = Long.MAX_VALUE;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
if (++count > maxChildren) {
break;
}
minValue = Math.min(minValue, values.nextValue());
hasValue = true;
}
@ -228,11 +249,15 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
boolean hasValue = false;
double minValue = Double.POSITIVE_INFINITY;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
if (++count > maxChildren) {
break;
}
minValue = Math.min(minValue, values.nextValue());
hasValue = true;
}
@ -246,23 +271,27 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
BytesRefBuilder value = null;
protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
BytesRefBuilder bytesRefBuilder = null;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
if (++count > maxChildren) {
break;
}
final BytesRef innerValue = values.binaryValue();
if (value == null) {
if (bytesRefBuilder == null) {
builder.copyBytes(innerValue);
value = builder;
bytesRefBuilder = builder;
} else {
final BytesRef min = value.get().compareTo(innerValue) <= 0 ? value.get() : innerValue;
final BytesRef min = bytesRefBuilder.get().compareTo(innerValue) <= 0 ? bytesRefBuilder.get() : innerValue;
if (min == innerValue) {
value.copyBytes(min);
bytesRefBuilder.copyBytes(min);
}
}
}
}
return value == null ? null : value.get();
return bytesRefBuilder == null ? null : bytesRefBuilder.get();
}
@Override
@ -271,16 +300,21 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
int ord = Integer.MAX_VALUE;
boolean hasValue = false;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
if (++count > maxChildren) {
break;
}
final int innerOrd = values.ordValue();
ord = Math.min(ord, innerOrd);
hasValue = true;
}
}
return hasValue ? ord : -1;
}
},
@ -299,13 +333,17 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
boolean hasValue = false;
long maxValue = Long.MIN_VALUE;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
final int count = values.docValueCount();
for (int i = 0; i < count - 1; ++i) {
if (++count > maxChildren) {
break;
}
final int docCount = values.docValueCount();
for (int i = 0; i < docCount - 1; ++i) {
values.nextValue();
}
maxValue = Math.max(maxValue, values.nextValue());
@ -325,13 +363,17 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
boolean hasValue = false;
double maxValue = Double.NEGATIVE_INFINITY;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
final int count = values.docValueCount();
for (int i = 0; i < count - 1; ++i) {
if (++count > maxChildren) {
break;
}
final int docCount = values.docValueCount();
for (int i = 0; i < docCount - 1; ++i) {
values.nextValue();
}
maxValue = Math.max(maxValue, values.nextValue());
@ -351,23 +393,27 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
BytesRefBuilder value = null;
protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
BytesRefBuilder bytesRefBuilder = null;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
if (++count > maxChildren) {
break;
}
final BytesRef innerValue = values.binaryValue();
if (value == null) {
if (bytesRefBuilder == null) {
builder.copyBytes(innerValue);
value = builder;
bytesRefBuilder = builder;
} else {
final BytesRef max = value.get().compareTo(innerValue) > 0 ? value.get() : innerValue;
final BytesRef max = bytesRefBuilder.get().compareTo(innerValue) > 0 ? bytesRefBuilder.get() : innerValue;
if (max == innerValue) {
value.copyBytes(max);
bytesRefBuilder.copyBytes(max);
}
}
}
}
return value == null ? null : value.get();
return bytesRefBuilder == null ? null : bytesRefBuilder.get();
}
@Override
@ -380,10 +426,14 @@ public enum MultiValueMode implements Writeable {
}
@Override
protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
int ord = -1;
int count = 0;
for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
if (values.advanceExact(doc)) {
if (++count > maxChildren) {
break;
}
ord = Math.max(ord, values.ordValue());
}
}
@ -458,7 +508,7 @@ public enum MultiValueMode implements Writeable {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc, int maxChildren) throws IOException {
if (parentDocs == null || childDocs == null) {
return FieldData.replaceMissing(DocValues.emptyNumeric(), missingValue);
}
@ -486,7 +536,7 @@ public enum MultiValueMode implements Writeable {
}
lastSeenParentDoc = parentDoc;
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc);
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren);
return true;
}
@ -502,7 +552,7 @@ public enum MultiValueMode implements Writeable {
};
}
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
throw new IllegalArgumentException("Unsupported sort mode: " + this);
}
@ -555,7 +605,7 @@ public enum MultiValueMode implements Writeable {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc, int maxChildren) throws IOException {
if (parentDocs == null || childDocs == null) {
return FieldData.replaceMissing(FieldData.emptyNumericDouble(), missingValue);
}
@ -580,7 +630,7 @@ public enum MultiValueMode implements Writeable {
}
lastSeenParentDoc = parentDoc;
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc);
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren);
return true;
}
@ -591,7 +641,7 @@ public enum MultiValueMode implements Writeable {
};
}
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
throw new IllegalArgumentException("Unsupported sort mode: " + this);
}
@ -663,7 +713,7 @@ public enum MultiValueMode implements Writeable {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc, int maxChildren) throws IOException {
if (parentDocs == null || childDocs == null) {
return select(FieldData.emptySortedBinary(), missingValue);
}
@ -692,7 +742,7 @@ public enum MultiValueMode implements Writeable {
}
lastSeenParentDoc = parentDoc;
lastEmittedValue = pick(selectedValues, builder, childDocs, firstChildDoc, parentDoc);
lastEmittedValue = pick(selectedValues, builder, childDocs, firstChildDoc, parentDoc, maxChildren);
if (lastEmittedValue == null) {
lastEmittedValue = missingValue;
}
@ -706,7 +756,7 @@ public enum MultiValueMode implements Writeable {
};
}
protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
throw new IllegalArgumentException("Unsupported sort mode: " + this);
}
@ -779,7 +829,7 @@ public enum MultiValueMode implements Writeable {
* NOTE: Calling the returned instance on docs that are not root docs is illegal
* The returned instance can only be evaluate the current and upcoming docs
*/
public SortedDocValues select(final SortedSetDocValues values, final BitSet parentDocs, final DocIdSetIterator childDocs) throws IOException {
public SortedDocValues select(final SortedSetDocValues values, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxChildren) throws IOException {
if (parentDocs == null || childDocs == null) {
return select(DocValues.emptySortedSet());
}
@ -817,7 +867,7 @@ public enum MultiValueMode implements Writeable {
}
docID = lastSeenParentDoc = parentDoc;
lastEmittedOrd = pick(selectedValues, childDocs, firstChildDoc, parentDoc);
lastEmittedOrd = pick(selectedValues, childDocs, firstChildDoc, parentDoc, maxChildren);
return lastEmittedOrd != -1;
}
@ -833,7 +883,7 @@ public enum MultiValueMode implements Writeable {
};
}
protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException {
protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc, int maxChildren) throws IOException {
throw new IllegalArgumentException("Unsupported sort mode: " + this);
}

8
server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
View File

@ -332,6 +332,14 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
final Nested nested;
if (nestedSort != null) {
if (context.indexVersionCreated().before(Version.V_6_5_0) && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
throw new QueryShardException(context,
"max_children is only supported on v6.5.0 or higher");
}
if (nestedSort.getNestedSort() != null && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
throw new QueryShardException(context,
"max_children is only supported on last level of nested sort");
}
// new nested sorts takes priority
nested = resolveNested(context, nestedSort);
} else {

13
server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
View File

@ -54,6 +54,7 @@ import org.elasticsearch.index.query.GeoValidationMethod;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryRewriteContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.MultiValueMode;
@ -633,6 +634,14 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
final Nested nested;
if (nestedSort != null) {
if (context.indexVersionCreated().before(Version.V_6_5_0) && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
throw new QueryShardException(context,
"max_children is only supported on v6.5.0 or higher");
}
if (nestedSort.getNestedSort() != null && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
throw new QueryShardException(context,
"max_children is only supported on last level of nested sort");
}
// new nested sorts takes priority
nested = resolveNested(context, nestedSort);
} else {
@ -672,8 +681,10 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
} else {
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
final int maxChildren = nested.getNestedSort() != null ?
nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
selectedValues = finalSortMode.select(distanceValues, Double.POSITIVE_INFINITY, rootDocs, innerDocs,
context.reader().maxDoc());
context.reader().maxDoc(), maxChildren);
}
return selectedValues.getRawDoubleValues();
}

35
server/src/main/java/org/elasticsearch/search/sort/NestedSortBuilder.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.sort;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -38,9 +39,11 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
public static final ParseField NESTED_FIELD = new ParseField("nested");
public static final ParseField PATH_FIELD = new ParseField("path");
public static final ParseField FILTER_FIELD = new ParseField("filter");
public static final ParseField MAX_CHILDREN_FIELD = new ParseField("max_children");
private final String path;
private QueryBuilder filter;
private int maxChildren = Integer.MAX_VALUE;
private NestedSortBuilder nestedSort;
public NestedSortBuilder(String path) {
@ -51,6 +54,11 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
path = in.readOptionalString();
filter = in.readOptionalNamedWriteable(QueryBuilder.class);
nestedSort = in.readOptionalWriteable(NestedSortBuilder::new);
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
maxChildren = in.readVInt();
} else {
maxChildren = Integer.MAX_VALUE;
}
}
public String getPath() {
@ -61,11 +69,18 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
return filter;
}
public int getMaxChildren() { return maxChildren; }
public NestedSortBuilder setFilter(final QueryBuilder filter) {
this.filter = filter;
return this;
}
public NestedSortBuilder setMaxChildren(final int maxChildren) {
this.maxChildren = maxChildren;
return this;
}
public NestedSortBuilder getNestedSort() {
return nestedSort;
}
@ -83,6 +98,11 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
out.writeOptionalString(path);
out.writeOptionalNamedWriteable(filter);
out.writeOptionalWriteable(nestedSort);
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
out.writeVInt(maxChildren);
} else {
out.writeVInt(Integer.MAX_VALUE);
}
}
@Override
@ -94,6 +114,11 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
if (filter != null) {
builder.field(FILTER_FIELD.getPreferredName(), filter);
}
if (maxChildren != Integer.MAX_VALUE) {
builder.field(MAX_CHILDREN_FIELD.getPreferredName(), maxChildren);
}
if (nestedSort != null) {
builder.field(NESTED_FIELD.getPreferredName(), nestedSort);
}
@ -104,6 +129,7 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
public static NestedSortBuilder fromXContent(XContentParser parser) throws IOException {
String path = null;
QueryBuilder filter = null;
int maxChildren = Integer.MAX_VALUE;
NestedSortBuilder nestedSort = null;
XContentParser.Token token = parser.currentToken();
@ -116,6 +142,8 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
path = parser.text();
} else if (currentName.equals(FILTER_FIELD.getPreferredName())) {
filter = parseNestedFilter(parser);
} else if (currentName.equals(MAX_CHILDREN_FIELD.getPreferredName())) {
maxChildren = parser.intValue();
} else if (currentName.equals(NESTED_FIELD.getPreferredName())) {
nestedSort = NestedSortBuilder.fromXContent(parser);
} else {
@ -129,7 +157,7 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
throw new IllegalArgumentException("malformed nested sort format, must start with an object");
}
return new NestedSortBuilder(path).setFilter(filter).setNestedSort(nestedSort);
return new NestedSortBuilder(path).setFilter(filter).setMaxChildren(maxChildren).setNestedSort(nestedSort);
}
@Override
@ -143,12 +171,13 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
NestedSortBuilder that = (NestedSortBuilder) obj;
return Objects.equals(path, that.path)
&& Objects.equals(filter, that.filter)
&& Objects.equals(maxChildren, that.maxChildren)
&& Objects.equals(nestedSort, that.nestedSort);
}
@Override
public int hashCode() {
return Objects.hash(path, filter, nestedSort);
return Objects.hash(path, filter, nestedSort, maxChildren);
}
public NestedSortBuilder rewrite(QueryRewriteContext ctx) throws IOException {
@ -164,7 +193,7 @@ public class NestedSortBuilder implements Writeable, ToXContentObject {
rewriteNested = nestedSort.rewrite(ctx);
}
if (rewriteFilter != this.filter || rewriteNested != this.nestedSort) {
return new NestedSortBuilder(this.path).setFilter(rewriteFilter).setNestedSort(rewriteNested);
return new NestedSortBuilder(this.path).setFilter(rewriteFilter).setMaxChildren(this.maxChildren).setNestedSort(rewriteNested);
} else {
return this;
}

8
server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
View File

@ -319,6 +319,14 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
final Nested nested;
if (nestedSort != null) {
if (context.indexVersionCreated().before(Version.V_6_5_0) && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
throw new QueryShardException(context,
"max_children is only supported on v6.5.0 or higher");
}
if (nestedSort.getNestedSort() != null && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
throw new QueryShardException(context,
"max_children is only supported on last level of nested sort");
}
// new nested sorts takes priority
nested = resolveNested(context, nestedSort);
} else {

2
server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
View File

@ -195,7 +195,7 @@ public abstract class SortBuilder<T extends SortBuilder<T>> implements NamedWrit
} else {
parentQuery = objectMapper.nestedTypeFilter();
}
return new Nested(context.bitsetFilter(parentQuery), childQuery);
return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort);
}
private static Query resolveNestedQuery(QueryShardContext context, NestedSortBuilder nestedSort, Query parentQuery) throws IOException {

2
server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
View File

@ -161,7 +161,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
protected Nested createNested(IndexSearcher searcher, Query parentFilter, Query childFilter) throws IOException {
BitsetFilterCache s = indexService.cache().bitsetFilterCache();
return new Nested(s.getBitSetProducer(parentFilter), childFilter);
return new Nested(s.getBitSetProducer(parentFilter), childFilter, null);
}
public void testEmpty() throws Exception {

56
server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java
View File

@ -109,7 +109,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedNumeric(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs);
verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
public void testMultiValuedLongs() throws Exception {
@ -147,7 +148,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedNumeric(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs);
verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
private void verifySortedNumeric(Supplier<SortedNumericDocValues> supplier, int maxDoc) throws IOException {
@ -210,11 +212,11 @@ public class MultiValueModeTests extends ESTestCase {
}
}
private void verifySortedNumeric(Supplier<SortedNumericDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException {
private void verifySortedNumeric(Supplier<SortedNumericDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs, int maxChildren) throws IOException {
for (long missingValue : new long[] { 0, randomLong() }) {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) {
SortedNumericDocValues values = supplier.get();
final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc);
final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc, maxChildren);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
assertTrue(selected.advanceExact(root));
@ -228,8 +230,12 @@ public class MultiValueModeTests extends ESTestCase {
expected = Long.MAX_VALUE;
}
int numValues = 0;
int count = 0;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
if (values.advanceExact(child)) {
if (++count > maxChildren) {
break;
}
for (int j = 0; j < values.docValueCount(); ++j) {
if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) {
expected += values.nextValue();
@ -285,7 +291,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedNumericDouble(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs);
verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
public void testMultiValuedDoubles() throws Exception {
@ -323,7 +330,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedNumericDouble(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs);
verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
private void verifySortedNumericDouble(Supplier<SortedNumericDoubleValues> supplier, int maxDoc) throws IOException {
@ -385,11 +393,11 @@ public class MultiValueModeTests extends ESTestCase {
}
}
private void verifySortedNumericDouble(Supplier<SortedNumericDoubleValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException {
private void verifySortedNumericDouble(Supplier<SortedNumericDoubleValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs, int maxChildren) throws IOException {
for (long missingValue : new long[] { 0, randomLong() }) {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) {
SortedNumericDoubleValues values = supplier.get();
final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc);
final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc, maxChildren);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
assertTrue(selected.advanceExact(root));
@ -403,8 +411,12 @@ public class MultiValueModeTests extends ESTestCase {
expected = Long.MAX_VALUE;
}
int numValues = 0;
int count = 0;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
if (values.advanceExact(child)) {
if (++count > maxChildren) {
break;
}
for (int j = 0; j < values.docValueCount(); ++j) {
if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) {
expected += values.nextValue();
@ -463,7 +475,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedBinary(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs);
verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
public void testMultiValuedStrings() throws Exception {
@ -501,7 +514,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedBinary(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs);
verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
private void verifySortedBinary(Supplier<SortedBinaryDocValues> supplier, int maxDoc) throws IOException {
@ -548,11 +562,11 @@ public class MultiValueModeTests extends ESTestCase {
}
}
private void verifySortedBinary(Supplier<SortedBinaryDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException {
private void verifySortedBinary(Supplier<SortedBinaryDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs, int maxChildren) throws IOException {
for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(randomAlphaOfLengthBetween(8, 8)) }) {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
SortedBinaryDocValues values = supplier.get();
final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc);
final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc, maxChildren);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
assertTrue(selected.advanceExact(root));
@ -560,8 +574,12 @@ public class MultiValueModeTests extends ESTestCase {
verifyBinaryValueCanCalledMoreThanOnce(selected, actual);
BytesRef expected = null;
int count = 0;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
if (values.advanceExact(child)) {
if (++count > maxChildren) {
break;
}
for (int j = 0; j < values.docValueCount(); ++j) {
if (expected == null) {
expected = BytesRef.deepCopyOf(values.nextValue());
@ -630,7 +648,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedSet(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedSet(multiValues, numDocs, rootDocs, innerDocs);
verifySortedSet(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedSet(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
public void testMultiValuedOrds() throws Exception {
@ -676,7 +695,8 @@ public class MultiValueModeTests extends ESTestCase {
verifySortedSet(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verifySortedSet(multiValues, numDocs, rootDocs, innerDocs);
verifySortedSet(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE);
verifySortedSet(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs));
}
private void verifySortedSet(Supplier<SortedSetDocValues> supplier, int maxDoc) throws IOException {
@ -715,10 +735,10 @@ public class MultiValueModeTests extends ESTestCase {
}
}
private void verifySortedSet(Supplier<SortedSetDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException {
private void verifySortedSet(Supplier<SortedSetDocValues> supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs, int maxChildren) throws IOException {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
SortedSetDocValues values = supplier.get();
final SortedDocValues selected = mode.select(values, rootDocs, new BitSetIterator(innerDocs, 0L));
final SortedDocValues selected = mode.select(values, rootDocs, new BitSetIterator(innerDocs, 0L), maxChildren);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
int actual = -1;
@ -727,8 +747,12 @@ public class MultiValueModeTests extends ESTestCase {
verifyOrdValueCanCalledMoreThanOnce(selected, actual);
}
int expected = -1;
int count = 0;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
if (values.advanceExact(child)) {
if (++count > maxChildren) {
break;
}
for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
if (expected == -1) {
expected = (int) ord;

46
server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
View File

@ -1423,9 +1423,16 @@ public class FieldSortIT extends ESIntegTestCase {
ensureGreen();
client().prepareIndex("test", "type", "1").setSource(jsonBuilder().startObject()
.startObject("nested")
.field("foo", "bar bar")
.endObject()
.startArray("nested")
.startObject().field("foo", "bar bar").endObject()
.startObject().field("foo", "abc abc").endObject()
.endArray()
.endObject()).execute().actionGet();
client().prepareIndex("test", "type", "2").setSource(jsonBuilder().startObject()
.startArray("nested")
.startObject().field("foo", "abc abc").endObject()
.startObject().field("foo", "cba bca").endObject()
.endArray()
.endObject()).execute().actionGet();
refresh();
@ -1436,11 +1443,27 @@ public class FieldSortIT extends ESIntegTestCase {
.execute().actionGet();
assertNoFailures(searchResponse);
SearchHit[] hits = searchResponse.getHits().getHits();
for (int i = 0; i < hits.length; ++i) {
assertThat(hits[i].getSortValues().length, is(1));
assertThat(hits[i].getSortValues()[0], is("bar"));
}
assertThat(hits.length, is(2));
assertThat(hits[0].getSortValues().length, is(1));
assertThat(hits[1].getSortValues().length, is(1));
assertThat(hits[0].getSortValues()[0], is("cba"));
assertThat(hits[1].getSortValues()[0], is("bar"));
// We sort on nested fields with max_children limit
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders
.fieldSort("nested.foo")
.setNestedSort(new NestedSortBuilder("nested").setMaxChildren(1))
.order(SortOrder.DESC))
.execute().actionGet();
assertNoFailures(searchResponse);
hits = searchResponse.getHits().getHits();
assertThat(hits.length, is(2));
assertThat(hits[0].getSortValues().length, is(1));
assertThat(hits[1].getSortValues().length, is(1));
assertThat(hits[0].getSortValues()[0], is("bar"));
assertThat(hits[1].getSortValues()[0], is("abc"));
// We sort on nested sub field
searchResponse = client().prepareSearch()
@ -1449,10 +1472,11 @@ public class FieldSortIT extends ESIntegTestCase {
.execute().actionGet();
assertNoFailures(searchResponse);
hits = searchResponse.getHits().getHits();
for (int i = 0; i < hits.length; ++i) {
assertThat(hits[i].getSortValues().length, is(1));
assertThat(hits[i].getSortValues()[0], is("bar bar"));
}
assertThat(hits.length, is(2));
assertThat(hits[0].getSortValues().length, is(1));
assertThat(hits[1].getSortValues().length, is(1));
assertThat(hits[0].getSortValues()[0], is("cba bca"));
assertThat(hits[1].getSortValues()[0], is("bar bar"));
}
public void testSortDuelBetweenSingleShardAndMultiShardIndex() throws Exception {