Added support for sorting by fields inside one or more nested objects.
The sorting by nested field support has the following parameters on top of the already existing sort options: nested_path - Defines the on what nested object to sort. The actual sort field must be a direct field inside this nested object. The default is to use the most immediate inherited nested object from the sort field. nested_filter - A filter the inner objects inside the nested path should match with in order for its field values to be taken into account by sorting. Common case is to repeat the query / filter inside the nested filter or query. By default no nested_filter is active. Either the highest (max) or lowest (min) inner object is picked for during sorting depending on the sort_mode being used. The sort_mode options avg and sum can still be used for number based fields inside nested objects. All the values for the sort field are taken into account for each nested object. Closes #2662
This commit is contained in:
Martijn van Groningen
parent
8db436f107
commit
303e87fb69
|
|
@ -0,0 +1,316 @@
|
|||
package org.elasticsearch.index.search.nested;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.search.DocIdSet;
|
||||
import org.apache.lucene.search.FieldComparator;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
||||
import org.elasticsearch.common.lucene.docset.DocIdSets;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class NestedFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
|
||||
|
||||
private final SortMode sortMode;
|
||||
private final IndexFieldData.XFieldComparatorSource wrappedSource;
|
||||
private final Filter rootDocumentsFilter;
|
||||
private final Filter innerDocumentsFilter;
|
||||
|
||||
public NestedFieldComparatorSource(SortMode sortMode, IndexFieldData.XFieldComparatorSource wrappedSource, Filter rootDocumentsFilter, Filter innerDocumentsFilter) {
|
||||
this.sortMode = sortMode;
|
||||
this.wrappedSource = wrappedSource;
|
||||
this.rootDocumentsFilter = rootDocumentsFilter;
|
||||
this.innerDocumentsFilter = innerDocumentsFilter;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
|
||||
// +1: have one spare slot for value comparison between inner documents.
|
||||
FieldComparator wrappedComparator = wrappedSource.newComparator(fieldname, numHits + 1, sortPos, reversed);
|
||||
switch (sortMode) {
|
||||
case MAX:
|
||||
return new NestedFieldComparator.Highest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
|
||||
case MIN:
|
||||
return new NestedFieldComparator.Lowest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
|
||||
default:
|
||||
throw new ElasticSearchIllegalArgumentException(
|
||||
String.format("Unsupported sort_mode[%s] for nested type", sortMode)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortField.Type reducedType() {
|
||||
return wrappedSource.reducedType();
|
||||
}
|
||||
}
|
||||
|
||||
abstract class NestedFieldComparator extends FieldComparator {
|
||||
|
||||
final Filter rootDocumentsFilter;
|
||||
final Filter innerDocumentsFilter;
|
||||
final int spareSlot;
|
||||
|
||||
FieldComparator wrappedComparator;
|
||||
FixedBitSet rootDocuments;
|
||||
FixedBitSet innerDocuments;
|
||||
|
||||
NestedFieldComparator(FieldComparator wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
|
||||
this.wrappedComparator = wrappedComparator;
|
||||
this.rootDocumentsFilter = rootDocumentsFilter;
|
||||
this.innerDocumentsFilter = innerDocumentsFilter;
|
||||
this.spareSlot = spareSlot;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compare(int slot1, int slot2) {
|
||||
return wrappedComparator.compare(slot1, slot2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setBottom(int slot) {
|
||||
wrappedComparator.setBottom(slot);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
|
||||
DocIdSet innerDocuments = innerDocumentsFilter.getDocIdSet(context, null);
|
||||
if (DocIdSets.isEmpty(innerDocuments)) {
|
||||
this.innerDocuments = null;
|
||||
} else if (innerDocuments instanceof FixedBitSet) {
|
||||
this.innerDocuments = (FixedBitSet) innerDocuments;
|
||||
} else {
|
||||
this.innerDocuments = DocIdSets.toFixedBitSet(innerDocuments.iterator(), context.reader().maxDoc());
|
||||
}
|
||||
DocIdSet rootDocuments = rootDocumentsFilter.getDocIdSet(context, null);
|
||||
if (DocIdSets.isEmpty(rootDocuments)) {
|
||||
this.rootDocuments = null;
|
||||
} else if (rootDocuments instanceof FixedBitSet) {
|
||||
this.rootDocuments = (FixedBitSet) rootDocuments;
|
||||
} else {
|
||||
this.rootDocuments = DocIdSets.toFixedBitSet(rootDocuments.iterator(), context.reader().maxDoc());
|
||||
}
|
||||
|
||||
wrappedComparator = wrappedComparator.setNextReader(context);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object value(int slot) {
|
||||
return wrappedComparator.value(slot);
|
||||
}
|
||||
|
||||
final static class Lowest extends NestedFieldComparator {
|
||||
|
||||
Lowest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) {
|
||||
super(wrappedComparator, parentFilter, childFilter, spareSlot);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int rootDoc) throws IOException {
|
||||
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// We need to copy the lowest value from all nested docs into slot.
|
||||
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
|
||||
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// We only need to emit a single cmp value for any matching nested doc
|
||||
int cmp = wrappedComparator.compareBottom(nestedDoc);
|
||||
if (cmp > 0) {
|
||||
return cmp;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return cmp;
|
||||
}
|
||||
int cmp1 = wrappedComparator.compareBottom(nestedDoc);
|
||||
if (cmp1 > 0) {
|
||||
return cmp1;
|
||||
} else {
|
||||
if (cmp1 == 0) {
|
||||
cmp = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int rootDoc) throws IOException {
|
||||
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// We need to copy the lowest value from all nested docs into slot.
|
||||
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
|
||||
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return;
|
||||
}
|
||||
wrappedComparator.copy(spareSlot, nestedDoc);
|
||||
wrappedComparator.copy(slot, nestedDoc);
|
||||
|
||||
while (true) {
|
||||
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return;
|
||||
}
|
||||
wrappedComparator.copy(spareSlot, nestedDoc);
|
||||
if (wrappedComparator.compare(spareSlot, slot) < 0) {
|
||||
wrappedComparator.copy(slot, nestedDoc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public int compareDocToValue(int rootDoc, Object value) throws IOException {
|
||||
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// We need to copy the lowest value from all nested docs into slot.
|
||||
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
|
||||
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// We only need to emit a single cmp value for any matching nested doc
|
||||
int cmp = wrappedComparator.compareBottom(nestedDoc);
|
||||
if (cmp > 0) {
|
||||
return cmp;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return cmp;
|
||||
}
|
||||
int cmp1 = wrappedComparator.compareDocToValue(nestedDoc, value);
|
||||
if (cmp1 > 0) {
|
||||
return cmp1;
|
||||
} else {
|
||||
if (cmp1 == 0) {
|
||||
cmp = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
final static class Highest extends NestedFieldComparator {
|
||||
|
||||
Highest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) {
|
||||
super(wrappedComparator, parentFilter, childFilter, spareSlot);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int rootDoc) throws IOException {
|
||||
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
|
||||
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmp = wrappedComparator.compareBottom(nestedDoc);
|
||||
if (cmp < 0) {
|
||||
return cmp;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return cmp;
|
||||
}
|
||||
int cmp1 = wrappedComparator.compareBottom(nestedDoc);
|
||||
if (cmp1 < 0) {
|
||||
return cmp1;
|
||||
} else {
|
||||
if (cmp1 == 0) {
|
||||
cmp = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int rootDoc) throws IOException {
|
||||
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
|
||||
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return;
|
||||
}
|
||||
wrappedComparator.copy(spareSlot, nestedDoc);
|
||||
wrappedComparator.copy(slot, nestedDoc);
|
||||
|
||||
while (true) {
|
||||
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return;
|
||||
}
|
||||
wrappedComparator.copy(spareSlot, nestedDoc);
|
||||
if (wrappedComparator.compare(spareSlot, slot) > 0) {
|
||||
wrappedComparator.copy(slot, nestedDoc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public int compareDocToValue(int rootDoc, Object value) throws IOException {
|
||||
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
|
||||
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmp = wrappedComparator.compareBottom(nestedDoc);
|
||||
if (cmp < 0) {
|
||||
return cmp;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
|
||||
if (nestedDoc >= rootDoc || nestedDoc == -1) {
|
||||
return cmp;
|
||||
}
|
||||
int cmp1 = wrappedComparator.compareDocToValue(nestedDoc, value);
|
||||
if (cmp1 < 0) {
|
||||
return cmp1;
|
||||
} else {
|
||||
if (cmp1 == 0) {
|
||||
cmp = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@
|
|||
package org.elasticsearch.search.sort;
|
||||
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.index.query.FilterBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
|
@ -38,6 +39,10 @@ public class FieldSortBuilder extends SortBuilder {
|
|||
|
||||
private String sortMode;
|
||||
|
||||
private FilterBuilder nestedFilter;
|
||||
|
||||
private String nestedPath;
|
||||
|
||||
/**
|
||||
* Constructs a new sort based on a document field.
|
||||
*
|
||||
|
|
@ -86,6 +91,25 @@ public class FieldSortBuilder extends SortBuilder {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the nested filter that the nested objects should match with in order to be taken into account
|
||||
* for sorting.
|
||||
*/
|
||||
public FieldSortBuilder setNestedFilter(FilterBuilder nestedFilter) {
|
||||
this.nestedFilter = nestedFilter;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sets the nested path if sorting occurs on a field that is inside a nested object. By default when sorting on a
|
||||
* field inside a nested object, the nearest upper nested object is selected as nested path.
|
||||
*/
|
||||
public FieldSortBuilder setNestedPath(String nestedPath) {
|
||||
this.nestedPath = nestedPath;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
builder.startObject(fieldName);
|
||||
|
|
@ -101,6 +125,12 @@ public class FieldSortBuilder extends SortBuilder {
|
|||
if (sortMode != null) {
|
||||
builder.field("sort_mode", sortMode);
|
||||
}
|
||||
if (nestedFilter != null) {
|
||||
builder.field("nested_filter", nestedFilter, params);
|
||||
}
|
||||
if (nestedPath != null) {
|
||||
builder.field("nested_path", nestedPath);
|
||||
}
|
||||
builder.endObject();
|
||||
return builder;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,13 +21,20 @@ package org.elasticsearch.search.sort;
|
|||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.ObjectMappers;
|
||||
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
|
||||
import org.elasticsearch.index.mapper.object.ObjectMapper;
|
||||
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
|
||||
import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
|
||||
import org.elasticsearch.search.SearchParseElement;
|
||||
import org.elasticsearch.search.SearchParseException;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
|
@ -71,7 +78,7 @@ public class SortParseElement implements SearchParseElement {
|
|||
if (token == XContentParser.Token.START_OBJECT) {
|
||||
addCompoundSortField(parser, context, sortFields);
|
||||
} else if (token == XContentParser.Token.VALUE_STRING) {
|
||||
addSortField(context, sortFields, parser.text(), false, false, null, null);
|
||||
addSortField(context, sortFields, parser.text(), false, false, null, null, null, null);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -106,6 +113,8 @@ public class SortParseElement implements SearchParseElement {
|
|||
String innerJsonName = null;
|
||||
boolean ignoreUnmapped = false;
|
||||
SortMode sortMode = null;
|
||||
Filter nestedFilter = null;
|
||||
String nestedPath = null;
|
||||
token = parser.nextToken();
|
||||
if (token == XContentParser.Token.VALUE_STRING) {
|
||||
String direction = parser.text();
|
||||
|
|
@ -114,7 +123,7 @@ public class SortParseElement implements SearchParseElement {
|
|||
} else if (direction.equals("desc")) {
|
||||
reverse = !SCORE_FIELD_NAME.equals(fieldName);
|
||||
}
|
||||
addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode);
|
||||
addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode, nestedPath, nestedFilter);
|
||||
} else {
|
||||
if (parsers.containsKey(fieldName)) {
|
||||
sortFields.add(parsers.get(fieldName).parse(parser, context));
|
||||
|
|
@ -137,17 +146,23 @@ public class SortParseElement implements SearchParseElement {
|
|||
ignoreUnmapped = parser.booleanValue();
|
||||
} else if ("sort_mode".equals(innerJsonName) || "sortMode".equals(innerJsonName)) {
|
||||
sortMode = SortMode.fromString(parser.text());
|
||||
} else if ("nested_path".equals(innerJsonName)) {
|
||||
nestedPath = parser.text();
|
||||
}
|
||||
} else if (token == XContentParser.Token.START_OBJECT) {
|
||||
if ("nested_filter".equals(innerJsonName)) {
|
||||
nestedFilter = context.queryParserService().parseInnerFilter(parser);
|
||||
}
|
||||
}
|
||||
}
|
||||
addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode);
|
||||
addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode, nestedPath, nestedFilter);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addSortField(SearchContext context, List<SortField> sortFields, String fieldName, boolean reverse, boolean ignoreUnmapped, @Nullable final String missing, SortMode sortMode) {
|
||||
private void addSortField(SearchContext context, List<SortField> sortFields, String fieldName, boolean reverse, boolean ignoreUnmapped, @Nullable final String missing, SortMode sortMode, String nestedPath, Filter nestedFilter) {
|
||||
if (SCORE_FIELD_NAME.equals(fieldName)) {
|
||||
if (reverse) {
|
||||
sortFields.add(SORT_SCORE_REVERSE);
|
||||
|
|
@ -183,9 +198,62 @@ public class SortParseElement implements SearchParseElement {
|
|||
sortMode = null;
|
||||
}
|
||||
if (sortMode == null) {
|
||||
sortMode = reverse ? SortMode.MAX : SortMode.MIN;
|
||||
sortMode = resolveDefaultSortMode(reverse);
|
||||
}
|
||||
sortFields.add(new SortField(fieldMapper.names().indexName(), context.fieldData().getForField(fieldMapper).comparatorSource(missing, sortMode), reverse));
|
||||
|
||||
IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper)
|
||||
.comparatorSource(missing, sortMode);
|
||||
ObjectMapper objectMapper;
|
||||
if (nestedPath != null) {
|
||||
ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath);
|
||||
if (objectMappers == null) {
|
||||
throw new ElasticSearchIllegalArgumentException("Invalid nested path");
|
||||
}
|
||||
objectMapper = objectMappers.mapper();
|
||||
} else {
|
||||
objectMapper = resolveClosestNestedObjectMapper(fieldName, context);
|
||||
}
|
||||
if (objectMapper != null && objectMapper.nested().isNested()) {
|
||||
Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE);
|
||||
Filter innerDocumentsFilter;
|
||||
if (nestedFilter != null) {
|
||||
innerDocumentsFilter = context.filterCache().cache(nestedFilter);
|
||||
} else {
|
||||
innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter());
|
||||
}
|
||||
// For now the nested sorting doesn't support SUM or AVG
|
||||
if (sortMode == SortMode.SUM || sortMode == SortMode.AVG) {
|
||||
sortMode = resolveDefaultSortMode(reverse);
|
||||
}
|
||||
fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter);
|
||||
}
|
||||
sortFields.add(new SortField(fieldMapper.names().indexName(), fieldComparatorSource, reverse));
|
||||
}
|
||||
}
|
||||
|
||||
private static SortMode resolveDefaultSortMode(boolean reverse) {
|
||||
return reverse ? SortMode.MAX : SortMode.MIN;
|
||||
}
|
||||
|
||||
private static ObjectMapper resolveClosestNestedObjectMapper(String fieldName, SearchContext context) {
|
||||
int indexOf = fieldName.lastIndexOf('.');
|
||||
if (indexOf == -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String objectPath = fieldName.substring(0, indexOf);
|
||||
ObjectMappers objectMappers = context.mapperService().objectMapper(objectPath);
|
||||
if (objectMappers == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (ObjectMapper objectMapper : objectMappers) {
|
||||
if (objectMapper.nested().isNested()) {
|
||||
return objectMapper;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,174 @@
|
|||
package org.elasticsearch.benchmark.search.nested;
|
||||
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
import org.elasticsearch.action.bulk.BulkResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.client.Requests;
|
||||
import org.elasticsearch.common.StopWatch;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.SizeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.search.sort.SortBuilders;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
|
||||
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
|
||||
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
|
||||
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
||||
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class NestedSearchBenchMark {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Settings settings = settingsBuilder()
|
||||
.put("index.engine.robin.refreshInterval", "-1")
|
||||
.put("gateway.type", "local")
|
||||
.put(SETTING_NUMBER_OF_SHARDS, 1)
|
||||
.put(SETTING_NUMBER_OF_REPLICAS, 0)
|
||||
.build();
|
||||
|
||||
Node node1 = nodeBuilder()
|
||||
.settings(settingsBuilder().put(settings).put("name", "node1"))
|
||||
.node();
|
||||
Client client = node1.client();
|
||||
|
||||
int count = (int) SizeValue.parseSizeValue("1m").singles();
|
||||
int nestedCount = 10;
|
||||
int rootDocs = count / nestedCount;
|
||||
int batch = 100;
|
||||
int queryWarmup = 5;
|
||||
int queryCount = 500;
|
||||
String indexName = "test";
|
||||
ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth()
|
||||
.setWaitForGreenStatus().execute().actionGet();
|
||||
if (clusterHealthResponse.isTimedOut()) {
|
||||
System.err.println("--> Timed out waiting for cluster health");
|
||||
}
|
||||
|
||||
try {
|
||||
client.admin().indices().prepareCreate(indexName)
|
||||
.addMapping("type", XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.startObject("type")
|
||||
.startObject("properties")
|
||||
.startObject("field1")
|
||||
.field("type", "integer")
|
||||
.endObject()
|
||||
.startObject("field2")
|
||||
.field("type", "nested")
|
||||
.startObject("properties")
|
||||
.startObject("field3")
|
||||
.field("type", "integer")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
).execute().actionGet();
|
||||
clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().execute().actionGet();
|
||||
if (clusterHealthResponse.isTimedOut()) {
|
||||
System.err.println("--> Timed out waiting for cluster health");
|
||||
}
|
||||
|
||||
StopWatch stopWatch = new StopWatch().start();
|
||||
|
||||
System.out.println("--> Indexing [" + rootDocs + "] root documents and [" + (rootDocs * nestedCount) + "] nested objects");
|
||||
long ITERS = rootDocs / batch;
|
||||
long i = 1;
|
||||
int counter = 0;
|
||||
for (; i <= ITERS; i++) {
|
||||
BulkRequestBuilder request = client.prepareBulk();
|
||||
for (int j = 0; j < batch; j++) {
|
||||
counter++;
|
||||
XContentBuilder doc = XContentFactory.jsonBuilder().startObject()
|
||||
.field("field1", counter)
|
||||
.startArray("field2");
|
||||
for (int k = 0; k < nestedCount; k++) {
|
||||
doc = doc.startObject()
|
||||
.field("field3", k)
|
||||
.endObject();
|
||||
}
|
||||
doc = doc.endArray();
|
||||
request.add(
|
||||
Requests.indexRequest(indexName).setType("type").setId(Integer.toString(counter)).setSource(doc)
|
||||
);
|
||||
}
|
||||
BulkResponse response = request.execute().actionGet();
|
||||
if (response.hasFailures()) {
|
||||
System.err.println("--> failures...");
|
||||
}
|
||||
if (((i * batch) % 10000) == 0) {
|
||||
System.out.println("--> Indexed " + (i * batch) + " took " + stopWatch.stop().lastTaskTime());
|
||||
stopWatch.start();
|
||||
}
|
||||
}
|
||||
System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (count * (1 + nestedCount))) / stopWatch.totalTime().secondsFrac()));
|
||||
} catch (Exception e) {
|
||||
System.out.println("--> Index already exists, ignoring indexing phase, waiting for green");
|
||||
clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet();
|
||||
if (clusterHealthResponse.isTimedOut()) {
|
||||
System.err.println("--> Timed out waiting for cluster health");
|
||||
}
|
||||
}
|
||||
client.admin().indices().prepareRefresh().execute().actionGet();
|
||||
System.out.println("--> Number of docs in index: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount());
|
||||
|
||||
NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats()
|
||||
.setJvm(true).execute().actionGet();
|
||||
System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted());
|
||||
System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed());
|
||||
|
||||
System.out.println("--> Running match_all with sorting on nested field");
|
||||
// run just the child query, warm up first
|
||||
for (int j = 0; j < queryWarmup; j++) {
|
||||
SearchResponse searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(
|
||||
SortBuilders.fieldSort("field2.field3")
|
||||
.setNestedPath("field2")
|
||||
.sortMode("avg")
|
||||
.order(SortOrder.ASC)
|
||||
)
|
||||
.execute().actionGet();
|
||||
if (j == 0) {
|
||||
System.out.println("--> Warmup took: " + searchResponse.getTook());
|
||||
}
|
||||
if (searchResponse.getHits().totalHits() != rootDocs) {
|
||||
System.err.println("--> mismatch on hits");
|
||||
}
|
||||
}
|
||||
|
||||
long totalQueryTime = 0;
|
||||
for (int j = 0; j < queryCount; j++) {
|
||||
SearchResponse searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(
|
||||
SortBuilders.fieldSort("field2.field3")
|
||||
.setNestedPath("field2")
|
||||
.sortMode("avg")
|
||||
.order(j % 2 == 0 ? SortOrder.ASC : SortOrder.DESC)
|
||||
)
|
||||
.execute().actionGet();
|
||||
|
||||
if (searchResponse.getHits().totalHits() != rootDocs) {
|
||||
System.err.println("--> mismatch on hits");
|
||||
}
|
||||
totalQueryTime += searchResponse.getTookInMillis();
|
||||
}
|
||||
System.out.println("--> Sorting by nested fields took: " + (totalQueryTime / queryCount) + "ms");
|
||||
|
||||
statsResponse = client.admin().cluster().prepareNodesStats()
|
||||
.setJvm(true).execute().actionGet();
|
||||
System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted());
|
||||
System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -26,11 +26,15 @@ import org.elasticsearch.action.get.GetResponse;
|
|||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.search.SearchType;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.query.FilterBuilders;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.facet.FacetBuilders;
|
||||
import org.elasticsearch.search.facet.filter.FilterFacet;
|
||||
import org.elasticsearch.search.facet.termsstats.TermsStatsFacet;
|
||||
import org.elasticsearch.search.sort.SortBuilders;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.elasticsearch.test.integration.AbstractNodesTests;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
|
|
@ -595,4 +599,291 @@ public class SimpleNestedTests extends AbstractNodesTests {
|
|||
// assertThat(explanation.getDetails()[1].getDescription(), equalTo("Child[1]"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleNestedSorting() throws Exception {
|
||||
client.admin().indices().prepareDelete().execute().actionGet();
|
||||
client.admin().indices().prepareCreate("test")
|
||||
.setSettings(settingsBuilder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.put("index.referesh_interval", -1)
|
||||
.build()
|
||||
)
|
||||
.addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
|
||||
.startObject("nested1")
|
||||
.field("type", "nested")
|
||||
.endObject()
|
||||
.endObject().endObject().endObject())
|
||||
.execute().actionGet();
|
||||
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
|
||||
|
||||
client.prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
|
||||
.field("field1", 1)
|
||||
.startArray("nested1")
|
||||
.startObject()
|
||||
.field("field1", 5)
|
||||
.endObject()
|
||||
.startObject()
|
||||
.field("field1", 4)
|
||||
.endObject()
|
||||
.endArray()
|
||||
.endObject()).execute().actionGet();
|
||||
client.prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject()
|
||||
.field("field1", 2)
|
||||
.startArray("nested1")
|
||||
.startObject()
|
||||
.field("field1", 1)
|
||||
.endObject()
|
||||
.startObject()
|
||||
.field("field1", 2)
|
||||
.endObject()
|
||||
.endArray()
|
||||
.endObject()).execute().actionGet();
|
||||
client.prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject()
|
||||
.field("field1", 3)
|
||||
.startArray("nested1")
|
||||
.startObject()
|
||||
.field("field1", 3)
|
||||
.endObject()
|
||||
.startObject()
|
||||
.field("field1", 4)
|
||||
.endObject()
|
||||
.endArray()
|
||||
.endObject()).execute().actionGet();
|
||||
client.admin().indices().prepareRefresh().execute().actionGet();
|
||||
|
||||
SearchResponse searchResponse = client.prepareSearch("test")
|
||||
.setTypes("type1")
|
||||
.setQuery(QueryBuilders.matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.ASC))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().hits()[0].id(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().hits()[0].sortValues()[0].toString(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().hits()[1].id(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().hits()[1].sortValues()[0].toString(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().hits()[2].id(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().hits()[2].sortValues()[0].toString(), equalTo("4"));
|
||||
|
||||
searchResponse = client.prepareSearch("test")
|
||||
.setTypes("type1")
|
||||
.setQuery(QueryBuilders.matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.DESC))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().hits()[0].id(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().hits()[0].sortValues()[0].toString(), equalTo("5"));
|
||||
assertThat(searchResponse.getHits().hits()[1].id(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().hits()[1].sortValues()[0].toString(), equalTo("4"));
|
||||
assertThat(searchResponse.getHits().hits()[2].id(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().hits()[2].sortValues()[0].toString(), equalTo("2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSortNestedWithNestedFilter() throws Exception {
|
||||
client.admin().indices().prepareDelete().execute().actionGet();
|
||||
client.admin().indices().prepareCreate("test")
|
||||
.setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0))
|
||||
.addMapping("type1", XContentFactory.jsonBuilder().startObject()
|
||||
.startObject("type1")
|
||||
.startObject("properties")
|
||||
.startObject("grand_parent_values").field("type", "long").endObject()
|
||||
.startObject("parent").field("type", "nested")
|
||||
.startObject("properties")
|
||||
.startObject("parent_values").field("type", "long").endObject()
|
||||
.startObject("child").field("type", "nested")
|
||||
.startObject("properties")
|
||||
.startObject("child_values").field("type", "long").endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject())
|
||||
.execute().actionGet();
|
||||
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
|
||||
|
||||
client.prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject()
|
||||
.field("grand_parent_values", 1l)
|
||||
.startObject("parent")
|
||||
.field("filter", false)
|
||||
.field("parent_values", 1l)
|
||||
.startObject("child")
|
||||
.field("filter", true)
|
||||
.field("child_values", 1l)
|
||||
.endObject()
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", 6l)
|
||||
.endObject()
|
||||
.endObject()
|
||||
.startObject("parent")
|
||||
.field("filter", true)
|
||||
.field("parent_values", 2l)
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", -1l)
|
||||
.endObject()
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", 5l)
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()).execute().actionGet();
|
||||
|
||||
client.prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject()
|
||||
.field("grand_parent_values", 2l)
|
||||
.startObject("parent")
|
||||
.field("filter", false)
|
||||
.field("parent_values", 2l)
|
||||
.startObject("child")
|
||||
.field("filter", true)
|
||||
.field("child_values", 2l)
|
||||
.endObject()
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", 4l)
|
||||
.endObject()
|
||||
.endObject()
|
||||
.startObject("parent")
|
||||
.field("parent_values", 3l)
|
||||
.field("filter", true)
|
||||
.startObject("child")
|
||||
.field("child_values", -2l)
|
||||
.field("filter", false)
|
||||
.endObject()
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", 3l)
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()).execute().actionGet();
|
||||
|
||||
client.prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject()
|
||||
.field("grand_parent_values", 3l)
|
||||
.startObject("parent")
|
||||
.field("parent_values", 3l)
|
||||
.field("filter", false)
|
||||
.startObject("child")
|
||||
.field("filter", true)
|
||||
.field("child_values", 3l)
|
||||
.endObject()
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", 1l)
|
||||
.endObject()
|
||||
.endObject()
|
||||
.startObject("parent")
|
||||
.field("parent_values", 4l)
|
||||
.field("filter", true)
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", -3l)
|
||||
.endObject()
|
||||
.startObject("child")
|
||||
.field("filter", false)
|
||||
.field("child_values", 1l)
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()).execute().actionGet();
|
||||
client.admin().indices().prepareRefresh().execute().actionGet();
|
||||
|
||||
// Without nested filter
|
||||
SearchResponse searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(
|
||||
SortBuilders.fieldSort("parent.child.child_values")
|
||||
.setNestedPath("parent.child")
|
||||
.order(SortOrder.ASC)
|
||||
)
|
||||
.execute().actionGet();
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getHits().length, equalTo(3));
|
||||
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("-3"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("-2"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("-1"));
|
||||
|
||||
// With nested filter
|
||||
searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(
|
||||
SortBuilders.fieldSort("parent.child.child_values")
|
||||
.setNestedPath("parent.child")
|
||||
.setNestedFilter(FilterBuilders.termFilter("parent.child.filter", true))
|
||||
.order(SortOrder.ASC)
|
||||
)
|
||||
.execute().actionGet();
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getHits().length, equalTo(3));
|
||||
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3"));
|
||||
|
||||
// Nested path should be automatically detected, expect same results as above search request
|
||||
searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(
|
||||
SortBuilders.fieldSort("parent.child.child_values")
|
||||
.setNestedFilter(FilterBuilders.termFilter("parent.child.filter", true))
|
||||
.order(SortOrder.ASC)
|
||||
)
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getHits().length, equalTo(3));
|
||||
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3"));
|
||||
|
||||
searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(
|
||||
SortBuilders.fieldSort("parent.parent_values")
|
||||
.setNestedPath("parent.child")
|
||||
.setNestedFilter(FilterBuilders.termFilter("parent.filter", false))
|
||||
.order(SortOrder.ASC)
|
||||
)
|
||||
.execute().actionGet();
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getHits().length, equalTo(3));
|
||||
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3"));
|
||||
|
||||
searchResponse = client.prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(
|
||||
SortBuilders.fieldSort("parent.child.child_values")
|
||||
.setNestedPath("parent.child")
|
||||
.setNestedFilter(FilterBuilders.termFilter("parent.filter", false))
|
||||
.order(SortOrder.ASC)
|
||||
)
|
||||
.execute().actionGet();
|
||||
assertThat(searchResponse.getHits().totalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getHits().length, equalTo(3));
|
||||
// TODO: If we expose ToChildBlockJoinQuery we can filter sort values based on a higher level nested objects
|
||||
// assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("3"));
|
||||
// assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("-3"));
|
||||
// assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
|
||||
// assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("-2"));
|
||||
// assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("1"));
|
||||
// assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("-1"));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -607,7 +607,7 @@ public class SimpleSortTests extends AbstractNodesTests {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testSortLongMVField() throws Exception {
|
||||
public void testSortMVField() throws Exception {
|
||||
try {
|
||||
client.admin().indices().prepareDelete("test").execute().actionGet();
|
||||
} catch (Exception e) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,263 @@
|
|||
package org.elasticsearch.test.unit.index.search.nested;
|
||||
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.StringField;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.search.join.ScoreMode;
|
||||
import org.apache.lucene.search.join.ToParentBlockJoinQuery;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.common.lucene.search.AndFilter;
|
||||
import org.elasticsearch.common.lucene.search.NotFilter;
|
||||
import org.elasticsearch.common.lucene.search.TermFilter;
|
||||
import org.elasticsearch.common.lucene.search.XFilteredQuery;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
|
||||
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
|
||||
import org.elasticsearch.test.unit.index.fielddata.AbstractFieldDataTests;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class NestedFieldComparatorTests extends AbstractFieldDataTests {
|
||||
|
||||
@Override
|
||||
protected FieldDataType getFieldDataType() {
|
||||
return new FieldDataType("string", ImmutableSettings.builder().put("format", "paged_bytes"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNestedSorting() throws Exception {
|
||||
List<Document> docs = new ArrayList<Document>();
|
||||
Document document = new Document();
|
||||
document.add(new StringField("field2", "a", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "b", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "c", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "a", Field.Store.NO));
|
||||
docs.add(document);
|
||||
writer.addDocuments(docs);
|
||||
writer.commit();
|
||||
|
||||
docs.clear();
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "c", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "d", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "e", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "b", Field.Store.NO));
|
||||
docs.add(document);
|
||||
writer.addDocuments(docs);
|
||||
|
||||
docs.clear();
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "e", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "f", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "g", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "c", Field.Store.NO));
|
||||
docs.add(document);
|
||||
writer.addDocuments(docs);
|
||||
|
||||
docs.clear();
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "g", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "h", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "F", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "i", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "F", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "d", Field.Store.NO));
|
||||
docs.add(document);
|
||||
writer.addDocuments(docs);
|
||||
writer.commit();
|
||||
|
||||
docs.clear();
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "i", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "F", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "j", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "F", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "k", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "F", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "f", Field.Store.NO));
|
||||
docs.add(document);
|
||||
writer.addDocuments(docs);
|
||||
|
||||
docs.clear();
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "k", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "l", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "m", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "g", Field.Store.NO));
|
||||
docs.add(document);
|
||||
writer.addDocuments(docs);
|
||||
|
||||
// This doc will not be included, because it doesn't have nested docs
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "h", Field.Store.NO));
|
||||
writer.addDocument(document);
|
||||
|
||||
docs.clear();
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "m", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "T", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "n", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "F", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("field2", "o", Field.Store.NO));
|
||||
document.add(new StringField("filter_1", "F", Field.Store.NO));
|
||||
docs.add(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("__type", "parent", Field.Store.NO));
|
||||
document.add(new StringField("field1", "i", Field.Store.NO));
|
||||
docs.add(document);
|
||||
writer.addDocuments(docs);
|
||||
writer.commit();
|
||||
|
||||
// Some garbage docs, just to check if the NestedFieldComparator can deal with this.
|
||||
document = new Document();
|
||||
document.add(new StringField("fieldXXX", "x", Field.Store.NO));
|
||||
writer.addDocument(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("fieldXXX", "x", Field.Store.NO));
|
||||
writer.addDocument(document);
|
||||
document = new Document();
|
||||
document.add(new StringField("fieldXXX", "x", Field.Store.NO));
|
||||
writer.addDocument(document);
|
||||
|
||||
SortMode sortMode = SortMode.MIN;
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false));
|
||||
PagedBytesIndexFieldData indexFieldData = getForField("field2");
|
||||
BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, sortMode);
|
||||
Filter parentFilter = new TermFilter(new Term("__type", "parent"));
|
||||
Filter childFilter = new NotFilter(parentFilter);
|
||||
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
|
||||
ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new CachingWrapperFilter(parentFilter), ScoreMode.None);
|
||||
|
||||
Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
|
||||
TopFieldDocs topDocs = searcher.search(query, 5, sort);
|
||||
assertThat(topDocs.totalHits, equalTo(7));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(5));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(3));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("a"));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(7));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("c"));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(11));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("e"));
|
||||
assertThat(topDocs.scoreDocs[3].doc, equalTo(15));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("g"));
|
||||
assertThat(topDocs.scoreDocs[4].doc, equalTo(19));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("i"));
|
||||
|
||||
sortMode = SortMode.MAX;
|
||||
nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
|
||||
sort = new Sort(new SortField("field2", nestedComparatorSource, true));
|
||||
topDocs = searcher.search(query, 5, sort);
|
||||
assertThat(topDocs.totalHits, equalTo(7));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(5));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(28));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("o"));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(23));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("m"));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(19));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("k"));
|
||||
assertThat(topDocs.scoreDocs[3].doc, equalTo(15));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("i"));
|
||||
assertThat(topDocs.scoreDocs[4].doc, equalTo(11));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("g"));
|
||||
|
||||
|
||||
childFilter = new AndFilter(Arrays.asList(new NotFilter(parentFilter), new TermFilter(new Term("filter_1", "T"))));
|
||||
nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
|
||||
query = new ToParentBlockJoinQuery(
|
||||
new XFilteredQuery(new MatchAllDocsQuery(), childFilter),
|
||||
new CachingWrapperFilter(parentFilter),
|
||||
ScoreMode.None
|
||||
);
|
||||
sort = new Sort(new SortField("field2", nestedComparatorSource, true));
|
||||
topDocs = searcher.search(query, 5, sort);
|
||||
assertThat(topDocs.totalHits, equalTo(6));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(5));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(23));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("m"));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(28));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("m"));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(11));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("g"));
|
||||
assertThat(topDocs.scoreDocs[3].doc, equalTo(15));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("g"));
|
||||
assertThat(topDocs.scoreDocs[4].doc, equalTo(7));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("e"));
|
||||
|
||||
searcher.getIndexReader().close();
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue