diff --git a/src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java b/src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java new file mode 100644 index 00000000000..5ec294075f3 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java @@ -0,0 +1,316 @@ +package org.elasticsearch.index.search.nested; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.lucene.docset.DocIdSets; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; + +import java.io.IOException; + +/** + */ +public class NestedFieldComparatorSource extends IndexFieldData.XFieldComparatorSource { + + private final SortMode sortMode; + private final IndexFieldData.XFieldComparatorSource wrappedSource; + private final Filter rootDocumentsFilter; + private final Filter innerDocumentsFilter; + + public NestedFieldComparatorSource(SortMode sortMode, IndexFieldData.XFieldComparatorSource wrappedSource, Filter rootDocumentsFilter, Filter innerDocumentsFilter) { + this.sortMode = sortMode; + this.wrappedSource = wrappedSource; + this.rootDocumentsFilter = rootDocumentsFilter; + this.innerDocumentsFilter = innerDocumentsFilter; + } + + @Override + public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + // +1: have one spare slot for value comparison between inner documents. + FieldComparator wrappedComparator = wrappedSource.newComparator(fieldname, numHits + 1, sortPos, reversed); + switch (sortMode) { + case MAX: + return new NestedFieldComparator.Highest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); + case MIN: + return new NestedFieldComparator.Lowest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); + default: + throw new ElasticSearchIllegalArgumentException( + String.format("Unsupported sort_mode[%s] for nested type", sortMode) + ); + } + } + + @Override + public SortField.Type reducedType() { + return wrappedSource.reducedType(); + } +} + +abstract class NestedFieldComparator extends FieldComparator { + + final Filter rootDocumentsFilter; + final Filter innerDocumentsFilter; + final int spareSlot; + + FieldComparator wrappedComparator; + FixedBitSet rootDocuments; + FixedBitSet innerDocuments; + + NestedFieldComparator(FieldComparator wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) { + this.wrappedComparator = wrappedComparator; + this.rootDocumentsFilter = rootDocumentsFilter; + this.innerDocumentsFilter = innerDocumentsFilter; + this.spareSlot = spareSlot; + } + + @Override + public int compare(int slot1, int slot2) { + return wrappedComparator.compare(slot1, slot2); + } + + @Override + public void setBottom(int slot) { + wrappedComparator.setBottom(slot); + } + + @Override + public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { + DocIdSet innerDocuments = innerDocumentsFilter.getDocIdSet(context, null); + if (DocIdSets.isEmpty(innerDocuments)) { + this.innerDocuments = null; + } else if (innerDocuments instanceof FixedBitSet) { + this.innerDocuments = (FixedBitSet) innerDocuments; + } else { + this.innerDocuments = DocIdSets.toFixedBitSet(innerDocuments.iterator(), context.reader().maxDoc()); + } + DocIdSet rootDocuments = rootDocumentsFilter.getDocIdSet(context, null); + if (DocIdSets.isEmpty(rootDocuments)) { + this.rootDocuments = null; + } else if (rootDocuments instanceof FixedBitSet) { + this.rootDocuments = (FixedBitSet) rootDocuments; + } else { + this.rootDocuments = DocIdSets.toFixedBitSet(rootDocuments.iterator(), context.reader().maxDoc()); + } + + wrappedComparator = wrappedComparator.setNextReader(context); + return this; + } + + @Override + public Object value(int slot) { + return wrappedComparator.value(slot); + } + + final static class Lowest extends NestedFieldComparator { + + Lowest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) { + super(wrappedComparator, parentFilter, childFilter, spareSlot); + } + + @Override + public int compareBottom(int rootDoc) throws IOException { + if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { + return 0; + } + + // We need to copy the lowest value from all nested docs into slot. + int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); + int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return 0; + } + + // We only need to emit a single cmp value for any matching nested doc + int cmp = wrappedComparator.compareBottom(nestedDoc); + if (cmp > 0) { + return cmp; + } + + while (true) { + nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return cmp; + } + int cmp1 = wrappedComparator.compareBottom(nestedDoc); + if (cmp1 > 0) { + return cmp1; + } else { + if (cmp1 == 0) { + cmp = 0; + } + } + } + } + + @Override + public void copy(int slot, int rootDoc) throws IOException { + if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { + return; + } + + // We need to copy the lowest value from all nested docs into slot. + int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); + int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return; + } + wrappedComparator.copy(spareSlot, nestedDoc); + wrappedComparator.copy(slot, nestedDoc); + + while (true) { + nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return; + } + wrappedComparator.copy(spareSlot, nestedDoc); + if (wrappedComparator.compare(spareSlot, slot) < 0) { + wrappedComparator.copy(slot, nestedDoc); + } + } + } + + @Override + @SuppressWarnings("unchecked") + public int compareDocToValue(int rootDoc, Object value) throws IOException { + if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { + return 0; + } + + // We need to copy the lowest value from all nested docs into slot. + int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); + int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return 0; + } + + // We only need to emit a single cmp value for any matching nested doc + int cmp = wrappedComparator.compareBottom(nestedDoc); + if (cmp > 0) { + return cmp; + } + + while (true) { + nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return cmp; + } + int cmp1 = wrappedComparator.compareDocToValue(nestedDoc, value); + if (cmp1 > 0) { + return cmp1; + } else { + if (cmp1 == 0) { + cmp = 0; + } + } + } + } + + } + + final static class Highest extends NestedFieldComparator { + + Highest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) { + super(wrappedComparator, parentFilter, childFilter, spareSlot); + } + + @Override + public int compareBottom(int rootDoc) throws IOException { + if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { + return 0; + } + + int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); + int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return 0; + } + + int cmp = wrappedComparator.compareBottom(nestedDoc); + if (cmp < 0) { + return cmp; + } + + while (true) { + nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return cmp; + } + int cmp1 = wrappedComparator.compareBottom(nestedDoc); + if (cmp1 < 0) { + return cmp1; + } else { + if (cmp1 == 0) { + cmp = 0; + } + } + } + } + + @Override + public void copy(int slot, int rootDoc) throws IOException { + if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { + return; + } + + int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); + int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return; + } + wrappedComparator.copy(spareSlot, nestedDoc); + wrappedComparator.copy(slot, nestedDoc); + + while (true) { + nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return; + } + wrappedComparator.copy(spareSlot, nestedDoc); + if (wrappedComparator.compare(spareSlot, slot) > 0) { + wrappedComparator.copy(slot, nestedDoc); + } + } + } + + @Override + @SuppressWarnings("unchecked") + public int compareDocToValue(int rootDoc, Object value) throws IOException { + if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { + return 0; + } + + int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); + int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return 0; + } + + int cmp = wrappedComparator.compareBottom(nestedDoc); + if (cmp < 0) { + return cmp; + } + + while (true) { + nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); + if (nestedDoc >= rootDoc || nestedDoc == -1) { + return cmp; + } + int cmp1 = wrappedComparator.compareDocToValue(nestedDoc, value); + if (cmp1 < 0) { + return cmp1; + } else { + if (cmp1 == 0) { + cmp = 0; + } + } + } + } + + } + +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index a9ba6184167..9519ddecc2a 100644 --- a/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.sort; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.FilterBuilder; import java.io.IOException; @@ -38,6 +39,10 @@ public class FieldSortBuilder extends SortBuilder { private String sortMode; + private FilterBuilder nestedFilter; + + private String nestedPath; + /** * Constructs a new sort based on a document field. * @@ -86,6 +91,25 @@ public class FieldSortBuilder extends SortBuilder { return this; } + /** + * Sets the nested filter that the nested objects should match with in order to be taken into account + * for sorting. + */ + public FieldSortBuilder setNestedFilter(FilterBuilder nestedFilter) { + this.nestedFilter = nestedFilter; + return this; + } + + + /** + * Sets the nested path if sorting occurs on a field that is inside a nested object. By default when sorting on a + * field inside a nested object, the nearest upper nested object is selected as nested path. + */ + public FieldSortBuilder setNestedPath(String nestedPath) { + this.nestedPath = nestedPath; + return this; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(fieldName); @@ -101,6 +125,12 @@ public class FieldSortBuilder extends SortBuilder { if (sortMode != null) { builder.field("sort_mode", sortMode); } + if (nestedFilter != null) { + builder.field("nested_filter", nestedFilter, params); + } + if (nestedPath != null) { + builder.field("nested_path", nestedPath); + } builder.endObject(); return builder; } diff --git a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java index fa02da2bde4..5ec8287060d 100644 --- a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java @@ -21,13 +21,20 @@ package org.elasticsearch.search.sort; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import org.apache.lucene.search.Filter; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.ObjectMappers; import org.elasticsearch.index.mapper.core.NumberFieldMapper; +import org.elasticsearch.index.mapper.object.ObjectMapper; +import org.elasticsearch.index.search.nested.NestedFieldComparatorSource; +import org.elasticsearch.index.search.nested.NonNestedDocsFilter; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.internal.SearchContext; @@ -71,7 +78,7 @@ public class SortParseElement implements SearchParseElement { if (token == XContentParser.Token.START_OBJECT) { addCompoundSortField(parser, context, sortFields); } else if (token == XContentParser.Token.VALUE_STRING) { - addSortField(context, sortFields, parser.text(), false, false, null, null); + addSortField(context, sortFields, parser.text(), false, false, null, null, null, null); } } } else { @@ -106,6 +113,8 @@ public class SortParseElement implements SearchParseElement { String innerJsonName = null; boolean ignoreUnmapped = false; SortMode sortMode = null; + Filter nestedFilter = null; + String nestedPath = null; token = parser.nextToken(); if (token == XContentParser.Token.VALUE_STRING) { String direction = parser.text(); @@ -114,7 +123,7 @@ public class SortParseElement implements SearchParseElement { } else if (direction.equals("desc")) { reverse = !SCORE_FIELD_NAME.equals(fieldName); } - addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode); + addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode, nestedPath, nestedFilter); } else { if (parsers.containsKey(fieldName)) { sortFields.add(parsers.get(fieldName).parse(parser, context)); @@ -137,17 +146,23 @@ public class SortParseElement implements SearchParseElement { ignoreUnmapped = parser.booleanValue(); } else if ("sort_mode".equals(innerJsonName) || "sortMode".equals(innerJsonName)) { sortMode = SortMode.fromString(parser.text()); + } else if ("nested_path".equals(innerJsonName)) { + nestedPath = parser.text(); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if ("nested_filter".equals(innerJsonName)) { + nestedFilter = context.queryParserService().parseInnerFilter(parser); } } } - addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode); + addSortField(context, sortFields, fieldName, reverse, ignoreUnmapped, missing, sortMode, nestedPath, nestedFilter); } } } } } - private void addSortField(SearchContext context, List sortFields, String fieldName, boolean reverse, boolean ignoreUnmapped, @Nullable final String missing, SortMode sortMode) { + private void addSortField(SearchContext context, List sortFields, String fieldName, boolean reverse, boolean ignoreUnmapped, @Nullable final String missing, SortMode sortMode, String nestedPath, Filter nestedFilter) { if (SCORE_FIELD_NAME.equals(fieldName)) { if (reverse) { sortFields.add(SORT_SCORE_REVERSE); @@ -183,9 +198,62 @@ public class SortParseElement implements SearchParseElement { sortMode = null; } if (sortMode == null) { - sortMode = reverse ? SortMode.MAX : SortMode.MIN; + sortMode = resolveDefaultSortMode(reverse); } - sortFields.add(new SortField(fieldMapper.names().indexName(), context.fieldData().getForField(fieldMapper).comparatorSource(missing, sortMode), reverse)); + + IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper) + .comparatorSource(missing, sortMode); + ObjectMapper objectMapper; + if (nestedPath != null) { + ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath); + if (objectMappers == null) { + throw new ElasticSearchIllegalArgumentException("Invalid nested path"); + } + objectMapper = objectMappers.mapper(); + } else { + objectMapper = resolveClosestNestedObjectMapper(fieldName, context); + } + if (objectMapper != null && objectMapper.nested().isNested()) { + Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE); + Filter innerDocumentsFilter; + if (nestedFilter != null) { + innerDocumentsFilter = context.filterCache().cache(nestedFilter); + } else { + innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter()); + } + // For now the nested sorting doesn't support SUM or AVG + if (sortMode == SortMode.SUM || sortMode == SortMode.AVG) { + sortMode = resolveDefaultSortMode(reverse); + } + fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter); + } + sortFields.add(new SortField(fieldMapper.names().indexName(), fieldComparatorSource, reverse)); } } + + private static SortMode resolveDefaultSortMode(boolean reverse) { + return reverse ? SortMode.MAX : SortMode.MIN; + } + + private static ObjectMapper resolveClosestNestedObjectMapper(String fieldName, SearchContext context) { + int indexOf = fieldName.lastIndexOf('.'); + if (indexOf == -1) { + return null; + } + + String objectPath = fieldName.substring(0, indexOf); + ObjectMappers objectMappers = context.mapperService().objectMapper(objectPath); + if (objectMappers == null) { + return null; + } + + for (ObjectMapper objectMapper : objectMappers) { + if (objectMapper.nested().isNested()) { + return objectMapper; + } + } + + return null; + } + } diff --git a/src/test/java/org/elasticsearch/benchmark/search/nested/NestedSearchBenchMark.java b/src/test/java/org/elasticsearch/benchmark/search/nested/NestedSearchBenchMark.java new file mode 100644 index 00000000000..19c80c560d9 --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/search/nested/NestedSearchBenchMark.java @@ -0,0 +1,174 @@ +package org.elasticsearch.benchmark.search.nested; + +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; +import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.SizeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.node.Node; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; + +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.node.NodeBuilder.nodeBuilder; + +/** + */ +public class NestedSearchBenchMark { + + public static void main(String[] args) throws Exception { + Settings settings = settingsBuilder() + .put("index.engine.robin.refreshInterval", "-1") + .put("gateway.type", "local") + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + + Node node1 = nodeBuilder() + .settings(settingsBuilder().put(settings).put("name", "node1")) + .node(); + Client client = node1.client(); + + int count = (int) SizeValue.parseSizeValue("1m").singles(); + int nestedCount = 10; + int rootDocs = count / nestedCount; + int batch = 100; + int queryWarmup = 5; + int queryCount = 500; + String indexName = "test"; + ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth() + .setWaitForGreenStatus().execute().actionGet(); + if (clusterHealthResponse.isTimedOut()) { + System.err.println("--> Timed out waiting for cluster health"); + } + + try { + client.admin().indices().prepareCreate(indexName) + .addMapping("type", XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("properties") + .startObject("field1") + .field("type", "integer") + .endObject() + .startObject("field2") + .field("type", "nested") + .startObject("properties") + .startObject("field3") + .field("type", "integer") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ).execute().actionGet(); + clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().execute().actionGet(); + if (clusterHealthResponse.isTimedOut()) { + System.err.println("--> Timed out waiting for cluster health"); + } + + StopWatch stopWatch = new StopWatch().start(); + + System.out.println("--> Indexing [" + rootDocs + "] root documents and [" + (rootDocs * nestedCount) + "] nested objects"); + long ITERS = rootDocs / batch; + long i = 1; + int counter = 0; + for (; i <= ITERS; i++) { + BulkRequestBuilder request = client.prepareBulk(); + for (int j = 0; j < batch; j++) { + counter++; + XContentBuilder doc = XContentFactory.jsonBuilder().startObject() + .field("field1", counter) + .startArray("field2"); + for (int k = 0; k < nestedCount; k++) { + doc = doc.startObject() + .field("field3", k) + .endObject(); + } + doc = doc.endArray(); + request.add( + Requests.indexRequest(indexName).setType("type").setId(Integer.toString(counter)).setSource(doc) + ); + } + BulkResponse response = request.execute().actionGet(); + if (response.hasFailures()) { + System.err.println("--> failures..."); + } + if (((i * batch) % 10000) == 0) { + System.out.println("--> Indexed " + (i * batch) + " took " + stopWatch.stop().lastTaskTime()); + stopWatch.start(); + } + } + System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (count * (1 + nestedCount))) / stopWatch.totalTime().secondsFrac())); + } catch (Exception e) { + System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); + clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet(); + if (clusterHealthResponse.isTimedOut()) { + System.err.println("--> Timed out waiting for cluster health"); + } + } + client.admin().indices().prepareRefresh().execute().actionGet(); + System.out.println("--> Number of docs in index: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount()); + + NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats() + .setJvm(true).execute().actionGet(); + System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted()); + System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); + + System.out.println("--> Running match_all with sorting on nested field"); + // run just the child query, warm up first + for (int j = 0; j < queryWarmup; j++) { + SearchResponse searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("field2.field3") + .setNestedPath("field2") + .sortMode("avg") + .order(SortOrder.ASC) + ) + .execute().actionGet(); + if (j == 0) { + System.out.println("--> Warmup took: " + searchResponse.getTook()); + } + if (searchResponse.getHits().totalHits() != rootDocs) { + System.err.println("--> mismatch on hits"); + } + } + + long totalQueryTime = 0; + for (int j = 0; j < queryCount; j++) { + SearchResponse searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("field2.field3") + .setNestedPath("field2") + .sortMode("avg") + .order(j % 2 == 0 ? SortOrder.ASC : SortOrder.DESC) + ) + .execute().actionGet(); + + if (searchResponse.getHits().totalHits() != rootDocs) { + System.err.println("--> mismatch on hits"); + } + totalQueryTime += searchResponse.getTookInMillis(); + } + System.out.println("--> Sorting by nested fields took: " + (totalQueryTime / queryCount) + "ms"); + + statsResponse = client.admin().cluster().prepareNodesStats() + .setJvm(true).execute().actionGet(); + System.out.println("--> Committed heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapCommitted()); + System.out.println("--> Used heap size: " + statsResponse.getNodes()[0].getJvm().getMem().getHeapUsed()); + } + +} diff --git a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java index 2a53a0552e4..f7782facbd3 100644 --- a/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java +++ b/src/test/java/org/elasticsearch/test/integration/nested/SimpleNestedTests.java @@ -26,11 +26,15 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.facet.FacetBuilders; import org.elasticsearch.search.facet.filter.FilterFacet; import org.elasticsearch.search.facet.termsstats.TermsStatsFacet; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -595,4 +599,291 @@ public class SimpleNestedTests extends AbstractNodesTests { // assertThat(explanation.getDetails()[1].getDescription(), equalTo("Child[1]")); } + @Test + public void testSimpleNestedSorting() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test") + .setSettings(settingsBuilder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.referesh_interval", -1) + .build() + ) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("nested1") + .field("type", "nested") + .endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("field1", 1) + .startArray("nested1") + .startObject() + .field("field1", 5) + .endObject() + .startObject() + .field("field1", 4) + .endObject() + .endArray() + .endObject()).execute().actionGet(); + client.prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() + .field("field1", 2) + .startArray("nested1") + .startObject() + .field("field1", 1) + .endObject() + .startObject() + .field("field1", 2) + .endObject() + .endArray() + .endObject()).execute().actionGet(); + client.prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() + .field("field1", 3) + .startArray("nested1") + .startObject() + .field("field1", 3) + .endObject() + .startObject() + .field("field1", 4) + .endObject() + .endArray() + .endObject()).execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + SearchResponse searchResponse = client.prepareSearch("test") + .setTypes("type1") + .setQuery(QueryBuilders.matchAllQuery()) + .addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.ASC)) + .execute().actionGet(); + + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().hits()[0].id(), equalTo("2")); + assertThat(searchResponse.getHits().hits()[0].sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().hits()[1].id(), equalTo("3")); + assertThat(searchResponse.getHits().hits()[1].sortValues()[0].toString(), equalTo("3")); + assertThat(searchResponse.getHits().hits()[2].id(), equalTo("1")); + assertThat(searchResponse.getHits().hits()[2].sortValues()[0].toString(), equalTo("4")); + + searchResponse = client.prepareSearch("test") + .setTypes("type1") + .setQuery(QueryBuilders.matchAllQuery()) + .addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.DESC)) + .execute().actionGet(); + + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().hits()[0].id(), equalTo("1")); + assertThat(searchResponse.getHits().hits()[0].sortValues()[0].toString(), equalTo("5")); + assertThat(searchResponse.getHits().hits()[1].id(), equalTo("3")); + assertThat(searchResponse.getHits().hits()[1].sortValues()[0].toString(), equalTo("4")); + assertThat(searchResponse.getHits().hits()[2].id(), equalTo("2")); + assertThat(searchResponse.getHits().hits()[2].sortValues()[0].toString(), equalTo("2")); + } + + @Test + public void testSortNestedWithNestedFilter() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test") + .setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) + .addMapping("type1", XContentFactory.jsonBuilder().startObject() + .startObject("type1") + .startObject("properties") + .startObject("grand_parent_values").field("type", "long").endObject() + .startObject("parent").field("type", "nested") + .startObject("properties") + .startObject("parent_values").field("type", "long").endObject() + .startObject("child").field("type", "nested") + .startObject("properties") + .startObject("child_values").field("type", "long").endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject()) + .execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject() + .field("grand_parent_values", 1l) + .startObject("parent") + .field("filter", false) + .field("parent_values", 1l) + .startObject("child") + .field("filter", true) + .field("child_values", 1l) + .endObject() + .startObject("child") + .field("filter", false) + .field("child_values", 6l) + .endObject() + .endObject() + .startObject("parent") + .field("filter", true) + .field("parent_values", 2l) + .startObject("child") + .field("filter", false) + .field("child_values", -1l) + .endObject() + .startObject("child") + .field("filter", false) + .field("child_values", 5l) + .endObject() + .endObject() + .endObject()).execute().actionGet(); + + client.prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject() + .field("grand_parent_values", 2l) + .startObject("parent") + .field("filter", false) + .field("parent_values", 2l) + .startObject("child") + .field("filter", true) + .field("child_values", 2l) + .endObject() + .startObject("child") + .field("filter", false) + .field("child_values", 4l) + .endObject() + .endObject() + .startObject("parent") + .field("parent_values", 3l) + .field("filter", true) + .startObject("child") + .field("child_values", -2l) + .field("filter", false) + .endObject() + .startObject("child") + .field("filter", false) + .field("child_values", 3l) + .endObject() + .endObject() + .endObject()).execute().actionGet(); + + client.prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject() + .field("grand_parent_values", 3l) + .startObject("parent") + .field("parent_values", 3l) + .field("filter", false) + .startObject("child") + .field("filter", true) + .field("child_values", 3l) + .endObject() + .startObject("child") + .field("filter", false) + .field("child_values", 1l) + .endObject() + .endObject() + .startObject("parent") + .field("parent_values", 4l) + .field("filter", true) + .startObject("child") + .field("filter", false) + .field("child_values", -3l) + .endObject() + .startObject("child") + .field("filter", false) + .field("child_values", 1l) + .endObject() + .endObject() + .endObject()).execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + // Without nested filter + SearchResponse searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("parent.child.child_values") + .setNestedPath("parent.child") + .order(SortOrder.ASC) + ) + .execute().actionGet(); + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getHits().length, equalTo(3)); + assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("3")); + assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("-3")); + assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("-2")); + assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("-1")); + + // With nested filter + searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("parent.child.child_values") + .setNestedPath("parent.child") + .setNestedFilter(FilterBuilders.termFilter("parent.child.filter", true)) + .order(SortOrder.ASC) + ) + .execute().actionGet(); + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getHits().length, equalTo(3)); + assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3")); + assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3")); + + // Nested path should be automatically detected, expect same results as above search request + searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("parent.child.child_values") + .setNestedFilter(FilterBuilders.termFilter("parent.child.filter", true)) + .order(SortOrder.ASC) + ) + .execute().actionGet(); + + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getHits().length, equalTo(3)); + assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3")); + assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3")); + + searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("parent.parent_values") + .setNestedPath("parent.child") + .setNestedFilter(FilterBuilders.termFilter("parent.filter", false)) + .order(SortOrder.ASC) + ) + .execute().actionGet(); + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getHits().length, equalTo(3)); + assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3")); + assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3")); + + searchResponse = client.prepareSearch() + .setQuery(matchAllQuery()) + .addSort( + SortBuilders.fieldSort("parent.child.child_values") + .setNestedPath("parent.child") + .setNestedFilter(FilterBuilders.termFilter("parent.filter", false)) + .order(SortOrder.ASC) + ) + .execute().actionGet(); + assertThat(searchResponse.getHits().totalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getHits().length, equalTo(3)); + // TODO: If we expose ToChildBlockJoinQuery we can filter sort values based on a higher level nested objects +// assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("3")); +// assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("-3")); +// assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); +// assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("-2")); +// assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("1")); +// assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("-1")); + } + } \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java index cd0c8a189fc..e147378d151 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java @@ -607,7 +607,7 @@ public class SimpleSortTests extends AbstractNodesTests { } @Test - public void testSortLongMVField() throws Exception { + public void testSortMVField() throws Exception { try { client.admin().indices().prepareDelete("test").execute().actionGet(); } catch (Exception e) { diff --git a/src/test/java/org/elasticsearch/test/unit/index/search/nested/NestedFieldComparatorTests.java b/src/test/java/org/elasticsearch/test/unit/index/search/nested/NestedFieldComparatorTests.java new file mode 100644 index 00000000000..92396abf1fb --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/index/search/nested/NestedFieldComparatorTests.java @@ -0,0 +1,263 @@ +package org.elasticsearch.test.unit.index.search.nested; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.*; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.search.join.ToParentBlockJoinQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.search.AndFilter; +import org.elasticsearch.common.lucene.search.NotFilter; +import org.elasticsearch.common.lucene.search.TermFilter; +import org.elasticsearch.common.lucene.search.XFilteredQuery; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; +import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; +import org.elasticsearch.index.search.nested.NestedFieldComparatorSource; +import org.elasticsearch.test.unit.index.fielddata.AbstractFieldDataTests; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +public class NestedFieldComparatorTests extends AbstractFieldDataTests { + + @Override + protected FieldDataType getFieldDataType() { + return new FieldDataType("string", ImmutableSettings.builder().put("format", "paged_bytes")); + } + + @Test + public void testNestedSorting() throws Exception { + List docs = new ArrayList(); + Document document = new Document(); + document.add(new StringField("field2", "a", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "b", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "c", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "a", Field.Store.NO)); + docs.add(document); + writer.addDocuments(docs); + writer.commit(); + + docs.clear(); + document = new Document(); + document.add(new StringField("field2", "c", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "d", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "e", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "b", Field.Store.NO)); + docs.add(document); + writer.addDocuments(docs); + + docs.clear(); + document = new Document(); + document.add(new StringField("field2", "e", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "f", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "g", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "c", Field.Store.NO)); + docs.add(document); + writer.addDocuments(docs); + + docs.clear(); + document = new Document(); + document.add(new StringField("field2", "g", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "h", Field.Store.NO)); + document.add(new StringField("filter_1", "F", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "i", Field.Store.NO)); + document.add(new StringField("filter_1", "F", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "d", Field.Store.NO)); + docs.add(document); + writer.addDocuments(docs); + writer.commit(); + + docs.clear(); + document = new Document(); + document.add(new StringField("field2", "i", Field.Store.NO)); + document.add(new StringField("filter_1", "F", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "j", Field.Store.NO)); + document.add(new StringField("filter_1", "F", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "k", Field.Store.NO)); + document.add(new StringField("filter_1", "F", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "f", Field.Store.NO)); + docs.add(document); + writer.addDocuments(docs); + + docs.clear(); + document = new Document(); + document.add(new StringField("field2", "k", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "l", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "m", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "g", Field.Store.NO)); + docs.add(document); + writer.addDocuments(docs); + + // This doc will not be included, because it doesn't have nested docs + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "h", Field.Store.NO)); + writer.addDocument(document); + + docs.clear(); + document = new Document(); + document.add(new StringField("field2", "m", Field.Store.NO)); + document.add(new StringField("filter_1", "T", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "n", Field.Store.NO)); + document.add(new StringField("filter_1", "F", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("field2", "o", Field.Store.NO)); + document.add(new StringField("filter_1", "F", Field.Store.NO)); + docs.add(document); + document = new Document(); + document.add(new StringField("__type", "parent", Field.Store.NO)); + document.add(new StringField("field1", "i", Field.Store.NO)); + docs.add(document); + writer.addDocuments(docs); + writer.commit(); + + // Some garbage docs, just to check if the NestedFieldComparator can deal with this. + document = new Document(); + document.add(new StringField("fieldXXX", "x", Field.Store.NO)); + writer.addDocument(document); + document = new Document(); + document.add(new StringField("fieldXXX", "x", Field.Store.NO)); + writer.addDocument(document); + document = new Document(); + document.add(new StringField("fieldXXX", "x", Field.Store.NO)); + writer.addDocument(document); + + SortMode sortMode = SortMode.MIN; + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); + PagedBytesIndexFieldData indexFieldData = getForField("field2"); + BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, sortMode); + Filter parentFilter = new TermFilter(new Term("__type", "parent")); + Filter childFilter = new NotFilter(parentFilter); + NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new CachingWrapperFilter(parentFilter), ScoreMode.None); + + Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); + TopFieldDocs topDocs = searcher.search(query, 5, sort); + assertThat(topDocs.totalHits, equalTo(7)); + assertThat(topDocs.scoreDocs.length, equalTo(5)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(3)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("a")); + assertThat(topDocs.scoreDocs[1].doc, equalTo(7)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("c")); + assertThat(topDocs.scoreDocs[2].doc, equalTo(11)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("e")); + assertThat(topDocs.scoreDocs[3].doc, equalTo(15)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("g")); + assertThat(topDocs.scoreDocs[4].doc, equalTo(19)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("i")); + + sortMode = SortMode.MAX; + nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + sort = new Sort(new SortField("field2", nestedComparatorSource, true)); + topDocs = searcher.search(query, 5, sort); + assertThat(topDocs.totalHits, equalTo(7)); + assertThat(topDocs.scoreDocs.length, equalTo(5)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(28)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("o")); + assertThat(topDocs.scoreDocs[1].doc, equalTo(23)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("m")); + assertThat(topDocs.scoreDocs[2].doc, equalTo(19)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("k")); + assertThat(topDocs.scoreDocs[3].doc, equalTo(15)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("i")); + assertThat(topDocs.scoreDocs[4].doc, equalTo(11)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("g")); + + + childFilter = new AndFilter(Arrays.asList(new NotFilter(parentFilter), new TermFilter(new Term("filter_1", "T")))); + nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + query = new ToParentBlockJoinQuery( + new XFilteredQuery(new MatchAllDocsQuery(), childFilter), + new CachingWrapperFilter(parentFilter), + ScoreMode.None + ); + sort = new Sort(new SortField("field2", nestedComparatorSource, true)); + topDocs = searcher.search(query, 5, sort); + assertThat(topDocs.totalHits, equalTo(6)); + assertThat(topDocs.scoreDocs.length, equalTo(5)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(23)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("m")); + assertThat(topDocs.scoreDocs[1].doc, equalTo(28)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("m")); + assertThat(topDocs.scoreDocs[2].doc, equalTo(11)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("g")); + assertThat(topDocs.scoreDocs[3].doc, equalTo(15)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("g")); + assertThat(topDocs.scoreDocs[4].doc, equalTo(7)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("e")); + + searcher.getIndexReader().close(); + } + +}