Preserve the order of nested documents in the Lucene index (#34225)

Today we reverse the initial order of the nested documents when we
index them in order to ensure that parents documents appear after
their children. This means that a query will always match nested documents
in the reverse order of their offsets in the source document.
Reversing all documents is not needed so this change ensures that parents
documents appear after their children without modifying the initial order
in each nested level. This allows to match children in the order of their
appearance in the source document which is a requirement to efficiently
implement #33587. Old indices created before this change will continue
to reverse the order of nested documents to ensure backwark compatibility.

server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java

@@ -24,6 +24,7 @@ import com.carrotsearch.hppc.ObjectObjectMap;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexSettings;
 
@@ -32,6 +33,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
@@ -454,13 +456,40 @@ public abstract class ParseContext implements Iterable<ParseContext.Document>{
         }
 
         void postParse() {
-            // reverse the order of docs for nested docs support, parent should be last
             if (documents.size() > 1) {
                 docsReversed = true;
-                Collections.reverse(documents);
+                if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0_alpha1)) {
+                    /**
+                     * For indices created on or after {@link Version#V_7_0_0_alpha1} we preserve the order
+                     * of the children while ensuring that parents appear after them.
+                     */
+                    List<Document> newDocs = reorderParent(documents);
+                    documents.clear();
+                    documents.addAll(newDocs);
+                } else {
+                    // reverse the order of docs for nested docs support, parent should be last
+                    Collections.reverse(documents);
+                }
             }
         }
 
+        /**
+         * Returns a copy of the provided {@link List} where parent documents appear
+         * after their children.
+         */
+        private List<Document> reorderParent(List<Document> docs) {
+            List<Document> newDocs = new ArrayList<>(docs.size());
+            LinkedList<Document> parents = new LinkedList<>();
+            for (Document doc : docs) {
+                while (parents.peek() != doc.getParent()){
+                    newDocs.add(parents.poll());
+                }
+                parents.add(0, doc);
+            }
+            newDocs.addAll(parents);
+            return newDocs;
+        }
+
         @Override
         public Iterator<Document> iterator() {
             return documents.iterator();

server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

@@ -30,6 +30,7 @@ import org.apache.lucene.search.TotalHits.Relation;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BitSet;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.document.DocumentField;
@@ -38,6 +39,7 @@ import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -344,6 +346,7 @@ public class FetchPhase implements SearchPhase {
         ObjectMapper current = nestedObjectMapper;
         String originalName = nestedObjectMapper.name();
         SearchHit.NestedIdentity nestedIdentity = null;
+        final IndexSettings indexSettings = context.getQueryShardContext().getIndexSettings();
         do {
             Query parentFilter;
             nestedParentObjectMapper = current.getParentObjectMapper(mapperService);
@@ -374,12 +377,32 @@ public class FetchPhase implements SearchPhase {
             BitSet parentBits = context.bitsetFilterCache().getBitSetProducer(parentFilter).getBitSet(subReaderContext);
 
             int offset = 0;
-            int nextParent = parentBits.nextSetBit(currentParent);
+            if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0_alpha1)) {
-            for (int docId = childIter.advance(currentParent + 1); docId < nextParent && docId != DocIdSetIterator.NO_MORE_DOCS;
+                /**
-                 docId = childIter.nextDoc()) {
+                 * Starts from the previous parent and finds the offset of the
-                offset++;
+                 * <code>nestedSubDocID</code> within the nested children. Nested documents
+                 * are indexed in the same order than in the source array so the offset
+                 * of the nested child is the number of nested document with the same parent
+                 * that appear before him.
+                 */
+                int previousParent = parentBits.prevSetBit(currentParent);
+                for (int docId = childIter.advance(previousParent + 1); docId < nestedSubDocId && docId != DocIdSetIterator.NO_MORE_DOCS;
+                        docId = childIter.nextDoc()) {
+                    offset++;
+                }
+                currentParent = nestedSubDocId;
+            } else {
+                /**
+                 * Nested documents are in reverse order in this version so we start from the current nested document
+                 * and find the number of documents with the same parent that appear after it.
+                 */
+                int nextParent = parentBits.nextSetBit(currentParent);
+                for (int docId = childIter.advance(currentParent + 1); docId < nextParent && docId != DocIdSetIterator.NO_MORE_DOCS;
+                        docId = childIter.nextDoc()) {
+                    offset++;
+                }
+                currentParent = nextParent;
             }
-            currentParent = nextParent;
             current = nestedObjectMapper = nestedParentObjectMapper;
             int currentPrefix = current == null ? 0 : current.name().length() + 1;
             nestedIdentity = new SearchHit.NestedIdentity(originalName.substring(currentPrefix), offset, nestedIdentity);

server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java

@@ -389,28 +389,28 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
         assertEquals(6, doc.docs().size());
 
         Document nested = doc.docs().get(0);
-        assertFieldValue(nested, "n1.n2.target", 7L);
+        assertFieldValue(nested, "n1.n2.target", 3L);
         assertFieldValue(nested, "n1.target");
         assertFieldValue(nested, "target");
 
-        nested = doc.docs().get(2);
+        nested = doc.docs().get(1);
         assertFieldValue(nested, "n1.n2.target", 5L);
         assertFieldValue(nested, "n1.target");
         assertFieldValue(nested, "target");
 
         nested = doc.docs().get(3);
-        assertFieldValue(nested, "n1.n2.target", 3L);
+        assertFieldValue(nested, "n1.n2.target", 7L);
         assertFieldValue(nested, "n1.target");
         assertFieldValue(nested, "target");
 
-        Document parent = doc.docs().get(1);
+        Document parent = doc.docs().get(2);
         assertFieldValue(parent, "target");
-        assertFieldValue(parent, "n1.target", 7L);
+        assertFieldValue(parent, "n1.target", 3L, 5L);
         assertFieldValue(parent, "n1.n2.target");
 
         parent = doc.docs().get(4);
         assertFieldValue(parent, "target");
-        assertFieldValue(parent, "n1.target", 3L, 5L);
+        assertFieldValue(parent, "n1.target", 7L);
         assertFieldValue(parent, "n1.n2.target");
 
         Document root = doc.docs().get(5);

server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java

@@ -21,6 +21,8 @@ package org.elasticsearch.index.mapper;
 
 import java.util.HashSet;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -33,6 +35,7 @@ import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -120,11 +123,11 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
 
         assertThat(doc.docs().size(), equalTo(3));
         assertThat(doc.docs().get(0).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString()));
-        assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("3"));
+        assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("1"));
-        assertThat(doc.docs().get(0).get("nested1.field2"), equalTo("4"));
+        assertThat(doc.docs().get(0).get("nested1.field2"), equalTo("2"));
         assertThat(doc.docs().get(1).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString()));
-        assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("1"));
+        assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("3"));
-        assertThat(doc.docs().get(1).get("nested1.field2"), equalTo("2"));
+        assertThat(doc.docs().get(1).get("nested1.field2"), equalTo("4"));
 
         assertThat(doc.docs().get(2).get("field"), equalTo("value"));
     }
@@ -160,20 +163,20 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
                 XContentType.JSON));
 
         assertThat(doc.docs().size(), equalTo(7));
-        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("6"));
+        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("2"));
         assertThat(doc.docs().get(0).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(0).get("field"), nullValue());
-        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("5"));
+        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("3"));
         assertThat(doc.docs().get(1).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(1).get("field"), nullValue());
-        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("4"));
+        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("1"));
         assertThat(doc.docs().get(2).get("nested1.nested2.field2"), nullValue());
         assertThat(doc.docs().get(2).get("field"), nullValue());
-        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("3"));
+        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("5"));
         assertThat(doc.docs().get(3).get("field"), nullValue());
-        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("2"));
+        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("6"));
         assertThat(doc.docs().get(4).get("field"), nullValue());
-        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("1"));
+        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("4"));
         assertThat(doc.docs().get(5).get("nested1.nested2.field2"), nullValue());
         assertThat(doc.docs().get(5).get("field"), nullValue());
         assertThat(doc.docs().get(6).get("field"), equalTo("value"));
@@ -212,21 +215,21 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
                 XContentType.JSON));
 
         assertThat(doc.docs().size(), equalTo(7));
-        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("6"));
+        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("2"));
         assertThat(doc.docs().get(0).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(0).get("field"), nullValue());
-        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("5"));
+        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("3"));
         assertThat(doc.docs().get(1).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(1).get("field"), nullValue());
-        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("4"));
+        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("1"));
-        assertThat(doc.docs().get(2).get("nested1.nested2.field2"), equalTo("5"));
+        assertThat(doc.docs().get(2).get("nested1.nested2.field2"), equalTo("2"));
         assertThat(doc.docs().get(2).get("field"), nullValue());
-        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("3"));
+        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("5"));
         assertThat(doc.docs().get(3).get("field"), nullValue());
-        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("2"));
+        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("6"));
         assertThat(doc.docs().get(4).get("field"), nullValue());
-        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("1"));
+        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("4"));
-        assertThat(doc.docs().get(5).get("nested1.nested2.field2"), equalTo("2"));
+        assertThat(doc.docs().get(5).get("nested1.nested2.field2"), equalTo("5"));
         assertThat(doc.docs().get(5).get("field"), nullValue());
         assertThat(doc.docs().get(6).get("field"), equalTo("value"));
         assertThat(doc.docs().get(6).get("nested1.field1"), nullValue());
@@ -264,21 +267,21 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
                 XContentType.JSON));
 
         assertThat(doc.docs().size(), equalTo(7));
-        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("6"));
+        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("2"));
         assertThat(doc.docs().get(0).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(0).get("field"), nullValue());
-        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("5"));
+        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("3"));
         assertThat(doc.docs().get(1).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(1).get("field"), nullValue());
-        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("4"));
+        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("1"));
-        assertThat(doc.docs().get(2).get("nested1.nested2.field2"), equalTo("5"));
+        assertThat(doc.docs().get(2).get("nested1.nested2.field2"), equalTo("2"));
         assertThat(doc.docs().get(2).get("field"), nullValue());
-        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("3"));
+        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("5"));
         assertThat(doc.docs().get(3).get("field"), nullValue());
-        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("2"));
+        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("6"));
         assertThat(doc.docs().get(4).get("field"), nullValue());
-        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("1"));
+        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("4"));
-        assertThat(doc.docs().get(5).get("nested1.nested2.field2"), equalTo("2"));
+        assertThat(doc.docs().get(5).get("nested1.nested2.field2"), equalTo("5"));
         assertThat(doc.docs().get(5).get("field"), nullValue());
         assertThat(doc.docs().get(6).get("field"), equalTo("value"));
         assertThat(doc.docs().get(6).getFields("nested1.field1").length, equalTo(2));
@@ -316,20 +319,20 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
                 XContentType.JSON));
 
         assertThat(doc.docs().size(), equalTo(7));
-        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("6"));
+        assertThat(doc.docs().get(0).get("nested1.nested2.field2"), equalTo("2"));
         assertThat(doc.docs().get(0).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(0).get("field"), nullValue());
-        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("5"));
+        assertThat(doc.docs().get(1).get("nested1.nested2.field2"), equalTo("3"));
         assertThat(doc.docs().get(1).get("nested1.field1"), nullValue());
         assertThat(doc.docs().get(1).get("field"), nullValue());
-        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("4"));
+        assertThat(doc.docs().get(2).get("nested1.field1"), equalTo("1"));
         assertThat(doc.docs().get(2).get("nested1.nested2.field2"), nullValue());
         assertThat(doc.docs().get(2).get("field"), nullValue());
-        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("3"));
+        assertThat(doc.docs().get(3).get("nested1.nested2.field2"), equalTo("5"));
         assertThat(doc.docs().get(3).get("field"), nullValue());
-        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("2"));
+        assertThat(doc.docs().get(4).get("nested1.nested2.field2"), equalTo("6"));
         assertThat(doc.docs().get(4).get("field"), nullValue());
-        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("1"));
+        assertThat(doc.docs().get(5).get("nested1.field1"), equalTo("4"));
         assertThat(doc.docs().get(5).get("nested1.nested2.field2"), nullValue());
         assertThat(doc.docs().get(5).get("field"), nullValue());
         assertThat(doc.docs().get(6).get("field"), equalTo("value"));
@@ -424,9 +427,9 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
                 XContentType.JSON));
 
         assertThat(doc.docs().size(), equalTo(3));
-        assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("4"));
+        assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("1"));
         assertThat(doc.docs().get(0).get("field"), nullValue());
-        assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("1"));
+        assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("4"));
         assertThat(doc.docs().get(1).get("field"), nullValue());
         assertThat(doc.docs().get(2).get("field"), equalTo("value"));
     }
@@ -634,4 +637,63 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
         );
     }
 
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        /**
+         * This is needed to force the index version with {@link IndexMetaData.SETTING_INDEX_VERSION_CREATED}.
+         */
+        return false;
+    }
+
+    public void testReorderParentBWC() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+            .startObject("nested1").field("type", "nested").endObject()
+            .endObject().endObject().endObject());
+
+        Version bwcVersion = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0,
+            Version.V_7_0_0_alpha1.minimumCompatibilityVersion());
+        for (Version version : new Version[] {Version.V_7_0_0_alpha1, bwcVersion}) {
+            DocumentMapper docMapper = createIndex("test-" + version,
+                Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), version).build())
+                    .mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+
+            assertThat(docMapper.hasNestedObjects(), equalTo(true));
+            ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
+            assertThat(nested1Mapper.nested().isNested(), equalTo(true));
+
+            ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
+                    .bytes(XContentFactory.jsonBuilder()
+                        .startObject()
+                        .field("field", "value")
+                        .startArray("nested1")
+                            .startObject()
+                                .field("field1", "1")
+                                .field("field2", "2")
+                            .endObject()
+                            .startObject()
+                                .field("field1", "3")
+                                .field("field2", "4")
+                            .endObject()
+                        .endArray()
+                        .endObject()),
+                XContentType.JSON));
+
+            assertThat(doc.docs().size(), equalTo(3));
+            if (version.onOrAfter(Version.V_7_0_0_alpha1)) {
+                assertThat(doc.docs().get(0).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString()));
+                assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("1"));
+                assertThat(doc.docs().get(0).get("nested1.field2"), equalTo("2"));
+                assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("3"));
+                assertThat(doc.docs().get(1).get("nested1.field2"), equalTo("4"));
+                assertThat(doc.docs().get(2).get("field"), equalTo("value"));
+            } else {
+                assertThat(doc.docs().get(0).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString()));
+                assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("3"));
+                assertThat(doc.docs().get(0).get("nested1.field2"), equalTo("4"));
+                assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("1"));
+                assertThat(doc.docs().get(1).get("nested1.field2"), equalTo("2"));
+                assertThat(doc.docs().get(2).get("field"), equalTo("value"));
+            }
+        }
+    }
 }

server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java

@@ -826,16 +826,16 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(topReviewers.getHits().getAt(2).getId(), equalTo("1"));
         assertThat(extractValue("name", topReviewers.getHits().getAt(2).getSourceAsMap()), equalTo("user c"));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getField().string(), equalTo("comments"));
-        assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getOffset(), equalTo(0));
+        assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getOffset(), equalTo(1));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
-        assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getOffset(), equalTo(2));
+        assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getOffset(), equalTo(0));
 
         assertThat(topReviewers.getHits().getAt(3).getId(), equalTo("1"));
         assertThat(extractValue("name", topReviewers.getHits().getAt(3).getSourceAsMap()), equalTo("user c"));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getField().string(), equalTo("comments"));
-        assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getOffset(), equalTo(1));
+        assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
-        assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getOffset(), equalTo(0));
+        assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getOffset(), equalTo(2));
 
         assertThat(topReviewers.getHits().getAt(4).getId(), equalTo("1"));
         assertThat(extractValue("name", topReviewers.getHits().getAt(4).getSourceAsMap()), equalTo("user d"));

server/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java

@@ -208,9 +208,9 @@ public class InnerHitsIT extends ESIntegTestCase {
         int size = randomIntBetween(0, numDocs);
         BoolQueryBuilder boolQuery = new BoolQueryBuilder();
         boolQuery.should(nestedQuery("field1", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder("a").setSize(size)
-                .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC))));
+                .addSort(new FieldSortBuilder("_doc").order(SortOrder.ASC))));
         boolQuery.should(nestedQuery("field2", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder("b")
-                .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)).setSize(size)));
+                .addSort(new FieldSortBuilder("_doc").order(SortOrder.ASC)).setSize(size)));
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(boolQuery)
                 .setSize(numDocs)

x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java

@@ -862,6 +862,9 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
                                 .startObject()
                                     .field("field2", "value2")
                                 .endObject()
+                                .startObject()
+                                    .array("field2", "value2", "value3")
+                                .endObject()
                             .endArray()
                         .endObject())
                 .get();
@@ -889,6 +892,9 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(0).getSourceAsString(),
                 equalTo("{\"field2\":\"value2\"}"));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(1).getNestedIdentity().getOffset(), equalTo(1));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("nested_field").getAt(1).getSourceAsString(),
+            equalTo("{\"field2\":[\"value2\",\"value3\"]}"));
     }
 
     public void testSuggesters() throws Exception {