Fielddata: goodbye comparators.

This commit removes custom comparators in favor of the ones that are in Lucene.

The major change is for nested documents: instead of having a comparator wrapper
that deals with nested documents, this is done at the fielddata level by having
a selector that returns the value to use for comparison.

Sorting with custom missing string values might be slower since it is using
TermValComparator since Lucene's TermOrdValComparator only supports sorting
missing values first or last. But other than this particular case, this change
will allow us to benefit from improvements on comparators from the Lucene side.

Close #5980

src/main/java/org/elasticsearch/index/fielddata/FieldData.java

@@ -40,6 +40,13 @@ public enum FieldData {
         assert Lucene.VERSION == Version.LUCENE_4_9 : "Remove emptySortedNumeric in 4.10 and use the method with the same name from Lucene's DocValues class. See LUCENE-5834.";
     }
 
+    /**
+     * Return a {@link SortedBinaryDocValues} that doesn't contain any value.
+     */
+    public static SortedBinaryDocValues emptySortedBinary(int maxDoc) {
+        return singleton(DocValues.emptyBinary(), new Bits.MatchNoBits(maxDoc));
+    }
+
     /**
      * Return a {@link SortedNumericDocValues} that doesn't contain any value.
      */

src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java

@@ -21,20 +21,23 @@ package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.*;
-import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.UnicodeUtil;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexComponent;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.settings.IndexSettings;
 import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.io.IOException;
+
 /**
  * Thread-safe utility class that allows to get per-segment values via the
  * {@link #load(AtomicReaderContext)} method.
@@ -93,7 +96,7 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
     /**
      * Comparator used for sorting.
      */
-    XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode);
+    XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested);
 
     /**
      * Clears any resources associated with this field data.
@@ -116,6 +119,52 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
             UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM);
         }
 
+        /**
+         * Simple wrapper class around a filter that matches parent documents
+         * and a filter that matches child documents. For every root document R,
+         * R will be in the parent filter and its children documents will be the
+         * documents that are contained in the inner set between the previous
+         * parent + 1, or 0 if there is no previous parent, and R (excluded).
+         */
+        public static class Nested {
+            private final Filter rootFilter, innerFilter;
+
+            public Nested(Filter rootFilter, Filter innerFilter) {
+                this.rootFilter = rootFilter;
+                this.innerFilter = innerFilter;
+            }
+
+            // TODO: nested docs should not be random filters but specialized
+            // ones that guarantee that you always get a FixedBitSet
+            @Deprecated
+            private static FixedBitSet toFixedBitSet(DocIdSet set, int maxDoc) throws IOException {
+                if (set == null || set instanceof FixedBitSet) {
+                    return (FixedBitSet) set;
+                } else {
+                    final FixedBitSet fixedBitSet = new FixedBitSet(maxDoc);
+                    final DocIdSetIterator it = set.iterator();
+                    if (it != null) {
+                        fixedBitSet.or(it);
+                    }
+                    return fixedBitSet;
+                }
+            }
+
+            /**
+             * Get a {@link FixedBitSet} that matches the root documents.
+             */
+            public FixedBitSet rootDocs(AtomicReaderContext ctx) throws IOException {
+                return toFixedBitSet(rootFilter.getDocIdSet(ctx, null), ctx.reader().maxDoc());
+            }
+
+            /**
+             * Get a {@link FixedBitSet} that matches the inner documents.
+             */
+            public FixedBitSet innerDocs(AtomicReaderContext ctx) throws IOException {
+                return toFixedBitSet(innerFilter.getDocIdSet(ctx, null), ctx.reader().maxDoc());
+            }
+        }
+
         /** Whether missing values should be sorted first. */
         protected final boolean sortMissingFirst(Object missingValue) {
             return "_first".equals(missingValue);

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java

@@ -19,37 +19,41 @@
 
 package org.elasticsearch.index.fielddata.fieldcomparator;
 
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.RandomAccessOrds;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.Version;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
 /**
+ * Comparator source for string/binary values.
  */
 public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
 
-    /** UTF-8 term containing a single code point: {@link Character#MAX_CODE_POINT} which will compare greater than all other index terms
-     *  since {@link Character#MAX_CODE_POINT} is a noncharacter and thus shouldn't appear in an index term. */
-    public static final BytesRef MAX_TERM;
-    static {
-        MAX_TERM = new BytesRef();
-        final char[] chars = Character.toChars(Character.MAX_CODE_POINT);
-        UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM);
-    }
-
     private final IndexFieldData<?> indexFieldData;
     private final MultiValueMode sortMode;
     private final Object missingValue;
+    private final Nested nested;
 
-    public BytesRefFieldComparatorSource(IndexFieldData<?> indexFieldData, Object missingValue, MultiValueMode sortMode) {
+    public BytesRefFieldComparatorSource(IndexFieldData<?> indexFieldData, Object missingValue, MultiValueMode sortMode, Nested nested) {
         this.indexFieldData = indexFieldData;
         this.sortMode = sortMode;
         this.missingValue = missingValue;
+        this.nested = nested;
     }
 
     @Override
@@ -57,15 +61,229 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat
         return SortField.Type.STRING;
     }
 
+    protected SortedBinaryDocValues getValues(AtomicReaderContext context) {
+        return indexFieldData.load(context).getBytesValues();
+    }
+
+    protected void setScorer(Scorer scorer) {}
+
     @Override
     public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+        assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
-        final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed);
 
+        final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed;
+        final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed);
         if (indexFieldData instanceof IndexOrdinalsFieldData) {
-            return new BytesRefOrdValComparator((IndexOrdinalsFieldData) indexFieldData, numHits, sortMode, missingBytes);
+            // The ordinal-based comparator only supports sorting missing values first or last so when
+            // a missing value is provided we fall back to the (slow) value-based comparator
+            // TODO: handle arbitrary missing values via a selector
+            if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) {
+                return new FieldComparator.TermOrdValComparator(numHits, null, sortMissingLast) {
+
+                    @Override
+                    protected SortedDocValues getSortedDocValues(AtomicReaderContext context, String field) throws IOException {
+                        final RandomAccessOrds values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues();
+                        final SortedDocValues selectedValues;
+                        if (nested == null) {
+                            selectedValues = sortMode.select(values);
+                        } else {
+                            final FixedBitSet rootDocs = nested.rootDocs(context);
+                            final FixedBitSet innerDocs = nested.innerDocs(context);
+                            selectedValues = sortMode.select(values, rootDocs, innerDocs);
+                        }
+                        return selectedValues;
+                    }
+
+                    public BytesRef value(int slot) {
+                        // TODO: When serializing the response to the coordinating node, we lose the information about
+                        // whether the comparator sorts missing docs first or last. We should fix it and let
+                        // TopDocs.merge deal with it (it knows how to)
+                        BytesRef value = super.value(slot);
+                        if (value == null) {
+                            value = missingBytes;
+                        }
+                        return value;
+                    }
+
+                };
+            }
         }
-        return new BytesRefValComparator(indexFieldData, numHits, sortMode, missingBytes);
+
+        final BytesRef nullPlaceHolder = new BytesRef();
+        final BytesRef nonNullMissingBytes = missingBytes == null ? nullPlaceHolder : missingBytes;
+        return new TermValComparator(numHits, null, sortMissingLast) {
+
+            @Override
+            protected BinaryDocValues getBinaryDocValues(AtomicReaderContext context, String field) throws IOException {
+                final SortedBinaryDocValues values = getValues(context);
+                final BinaryDocValues selectedValues;
+                if (nested == null) {
+                    selectedValues = sortMode.select(values, nonNullMissingBytes);
+                } else {
+                    final FixedBitSet rootDocs = nested.rootDocs(context);
+                    final FixedBitSet innerDocs = nested.innerDocs(context);
+                    selectedValues = sortMode.select(values, nonNullMissingBytes, rootDocs, innerDocs, context.reader().maxDoc());
+                }
+                return selectedValues;
+            }
+
+            @Override
+            protected Bits getDocsWithField(AtomicReaderContext context, String field) throws IOException {
+                return new Bits.MatchAllBits(context.reader().maxDoc());
+            }
+
+            @Override
+            protected boolean isNull(int doc, BytesRef term) {
+                return term == nullPlaceHolder;
+            }
+
+            @Override
+            public void setScorer(Scorer scorer) {
+                BytesRefFieldComparatorSource.this.setScorer(scorer);
+            }
+
+            @Override
+            public BytesRef value(int slot) {
+                BytesRef value = super.value(slot);
+                if (value == null) {
+                    value = missingBytes;
+                }
+                return value;
+            }
+
+        };
+    }
+
+    static {
+        assert Lucene.VERSION == Version.LUCENE_4_9 : "The comparator below is a raw copy of Lucene's, remove it when upgrading to 4.10";
+    }
+
+    /** Sorts by field's natural Term sort order.  All
+     *  comparisons are done using BytesRef.compareTo, which is
+     *  slow for medium to large result sets but possibly
+     *  very fast for very small results sets. */
+    public static class TermValComparator extends FieldComparator<BytesRef> {
+
+      private final BytesRef[] values;
+      private final BytesRef[] tempBRs;
+      private BinaryDocValues docTerms;
+      private Bits docsWithField;
+      private final String field;
+      private BytesRef bottom;
+      private BytesRef topValue;
+      private final int missingSortCmp;
+
+      /** Sole constructor. */
+      public TermValComparator(int numHits, String field, boolean sortMissingLast) {
+        values = new BytesRef[numHits];
+        tempBRs = new BytesRef[numHits];
+        this.field = field;
+        missingSortCmp = sortMissingLast ? 1 : -1;
+      }
+
+      @Override
+      public int compare(int slot1, int slot2) {
+        final BytesRef val1 = values[slot1];
+        final BytesRef val2 = values[slot2];
+        return compareValues(val1, val2);
+      }
+
+      @Override
+      public int compareBottom(int doc) {
+        final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
+        return compareValues(bottom, comparableBytes);
+      }
+
+      @Override
+      public void copy(int slot, int doc) {
+        final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
+        if (comparableBytes == null) {
+          values[slot] = null;
+        } else {
+          if (tempBRs[slot] == null) {
+            tempBRs[slot] = new BytesRef();
+          }
+          values[slot] = tempBRs[slot];
+          values[slot].copyBytes(comparableBytes);
+        }
+      }
+
+      /** Retrieves the BinaryDocValues for the field in this segment */
+      protected BinaryDocValues getBinaryDocValues(AtomicReaderContext context, String field) throws IOException {
+        return FieldCache.DEFAULT.getTerms(context.reader(), field, true);
+      }
+
+      /** Retrieves the set of documents that have a value in this segment */
+      protected Bits getDocsWithField(AtomicReaderContext context, String field) throws IOException {
+        return FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
+      }
+
+      /** Check whether the given value represents <tt>null</tt>. This can be
+       *  useful if the {@link BinaryDocValues} returned by {@link #getBinaryDocValues}
+       *  use a special value as a sentinel. The default implementation checks
+       *  {@link #getDocsWithField}.
+       *  <p>NOTE: The null value can only be an EMPTY {@link BytesRef}. */
+      protected boolean isNull(int doc, BytesRef term) {
+        return docsWithField != null && docsWithField.get(doc) == false;
+      }
+
+      @Override
+      public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+        docTerms = getBinaryDocValues(context, field);
+        docsWithField = getDocsWithField(context, field);
+        if (docsWithField instanceof Bits.MatchAllBits) {
+          docsWithField = null;
+        }
+        return this;
+      }
+
+      @Override
+      public void setBottom(final int bottom) {
+        this.bottom = values[bottom];
+      }
+
+      @Override
+      public void setTopValue(BytesRef value) {
+        // null is fine: it means the last doc of the prior
+        // search was missing this value
+        topValue = value;
+      }
+
+      @Override
+      public BytesRef value(int slot) {
+        return values[slot];
+      }
+
+      @Override
+      public int compareValues(BytesRef val1, BytesRef val2) {
+        // missing always sorts first:
+        if (val1 == null) {
+          if (val2 == null) {
+            return 0;
+          }
+          return missingSortCmp;
+        } else if (val2 == null) {
+          return -missingSortCmp;
+        }
+        return val1.compareTo(val2);
+      }
+
+      @Override
+      public int compareTop(int doc) {
+        final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
+        return compareValues(topValue, comparableBytes);
+      }
+
+      /**
+       * Given a document and a term, return the term itself if it exists or
+       * <tt>null</tt> otherwise.
+       */
+      private BytesRef getComparableBytes(int doc, BytesRef term) {
+        if (term.length == 0 && isNull(doc, term)) {
+          return null;
+        }
+        return term;
+      }
     }
 
 }

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java

@@ -1,339 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-
-/**
- * Sorts by field's natural Term sort order, using
- * ordinals.  This is functionally equivalent to {@link
- * org.apache.lucene.search.FieldComparator.TermValComparator}, but it first resolves the string
- * to their relative ordinal positions (using the index
- * returned by {@link org.apache.lucene.search.FieldCache#getTermsIndex}), and
- * does most comparisons using the ordinals.  For medium
- * to large results, this comparator will be much faster
- * than {@link org.apache.lucene.search.FieldComparator.TermValComparator}.  For very small
- * result sets it may be slower.
- *
- * Internally this comparator multiplies ordinals by 4 so that virtual ordinals can be inserted in-between the original field data ordinals.
- * Thanks to this, an ordinal for the missing value and the bottom value can be computed and all ordinals are directly comparable. For example,
- * if the field data ordinals are (a,1), (b,2) and (c,3), they will be internally stored as (a,4), (b,8), (c,12). Then the ordinal for the
- * missing value will be computed by binary searching. For example, if the missing value is 'ab', it will be assigned 6 as an ordinal (between
- * 'a' and 'b'. And if the bottom value is 'ac', it will be assigned 7 as an ordinal (between 'ab' and 'b').
- */
-public final class BytesRefOrdValComparator extends NestedWrappableComparator<BytesRef> {
-
-    final IndexOrdinalsFieldData indexFieldData;
-    final BytesRef missingValue;
-
-    /* Ords for each slot, times 4.
-       @lucene.internal */
-    final long[] ords;
-
-    final MultiValueMode sortMode;
-
-    /* Values for each slot.
-       @lucene.internal */
-    final BytesRef[] values;
-
-    /* Which reader last copied a value into the slot. When
-       we compare two slots, we just compare-by-ord if the
-       readerGen is the same; else we must compare the
-       values (slower).
-       @lucene.internal */
-    final int[] readerGen;
-
-    /* Gen of current reader we are on.
-       @lucene.internal */
-    int currentReaderGen = -1;
-
-    /* Current reader's doc ord/values.
-       @lucene.internal */
-    SortedDocValues termsIndex;
-    long missingOrd;
-
-    /* Bottom slot, or -1 if queue isn't full yet
-       @lucene.internal */
-    int bottomSlot = -1;
-
-    /* Bottom ord (same as ords[bottomSlot] once bottomSlot
-       is set).  Cached for faster compares.
-       @lucene.internal */
-    long bottomOrd;
-
-    BytesRef top;
-    long topOrd;
-
-    public BytesRefOrdValComparator(IndexOrdinalsFieldData indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) {
-        this.indexFieldData = indexFieldData;
-        this.sortMode = sortMode;
-        this.missingValue = missingValue;
-        ords = new long[numHits];
-        values = new BytesRef[numHits];
-        readerGen = new int[numHits];
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        if (readerGen[slot1] == readerGen[slot2]) {
-            final int res = Long.compare(ords[slot1], ords[slot2]);
-            assert Integer.signum(res) == Integer.signum(compareValues(values[slot1], values[slot2])) : values[slot1] + " " + values[slot2] + " " + ords[slot1] + " " + ords[slot2];
-            return res;
-        }
-
-        final BytesRef val1 = values[slot1];
-        final BytesRef val2 = values[slot2];
-        return compareValues(val1, val2);
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int compareTop(int doc) throws IOException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int compareBottomMissing() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void missing(int slot) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int compareTopMissing() {
-        throw new UnsupportedOperationException();
-    }
-
-    class PerSegmentComparator extends NestedWrappableComparator<BytesRef> {
-        final SortedDocValues termsIndex;
-
-        public PerSegmentComparator(SortedDocValues termsIndex) {
-            this.termsIndex = termsIndex;
-        }
-
-        @Override
-        public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-            return BytesRefOrdValComparator.this.setNextReader(context);
-        }
-
-        @Override
-        public int compare(int slot1, int slot2) {
-            return BytesRefOrdValComparator.this.compare(slot1, slot2);
-        }
-
-        @Override
-        public void setBottom(final int bottom) {
-            BytesRefOrdValComparator.this.setBottom(bottom);
-        }
-
-        @Override
-        public void setTopValue(BytesRef value) {
-            BytesRefOrdValComparator.this.setTopValue(value);
-        }
-
-        @Override
-        public BytesRef value(int slot) {
-            return BytesRefOrdValComparator.this.value(slot);
-        }
-
-        @Override
-        public int compareValues(BytesRef val1, BytesRef val2) {
-            if (val1 == null) {
-                if (val2 == null) {
-                    return 0;
-                }
-                return -1;
-            } else if (val2 == null) {
-                return 1;
-            }
-            return val1.compareTo(val2);
-        }
-
-        @Override
-        public int compareBottom(int doc) {
-            assert bottomSlot != -1;
-            final long docOrd = termsIndex.getOrd(doc);
-            final long comparableOrd = docOrd < 0 ? missingOrd : docOrd << 2;
-            return Long.compare(bottomOrd, comparableOrd);
-        }
-
-        @Override
-        public int compareTop(int doc) throws IOException {
-            final long ord = termsIndex.getOrd(doc);
-            if (ord < 0) {
-                return compareTopMissing();
-            } else {
-                final long comparableOrd = ord << 2;
-                return Long.compare(topOrd, comparableOrd);
-            }
-        }
-
-        @Override
-        public int compareBottomMissing() {
-            assert bottomSlot != -1;
-            return Long.compare(bottomOrd, missingOrd);
-        }
-
-        @Override
-        public int compareTopMissing() {
-            int cmp = Long.compare(topOrd, missingOrd);
-            if (cmp == 0) {
-                return compareValues(top, missingValue);
-            } else {
-                return cmp;
-            }
-        }
-
-        @Override
-        public void copy(int slot, int doc) {
-            final int ord = termsIndex.getOrd(doc);
-            if (ord < 0) {
-                ords[slot] = missingOrd;
-                values[slot] = missingValue;
-            } else {
-                assert ord >= 0;
-                ords[slot] = ((long) ord) << 2;
-                if (values[slot] == null || values[slot] == missingValue) {
-                    values[slot] = new BytesRef();
-                }
-                values[slot].copyBytes(termsIndex.lookupOrd(ord));
-            }
-            readerGen[slot] = currentReaderGen;
-        }
-
-        @Override
-        public void missing(int slot) {
-            ords[slot] = missingOrd;
-            values[slot] = missingValue;
-            readerGen[slot] = currentReaderGen;
-        }
-    }
-
-    // for assertions
-    private boolean consistentInsertedOrd(SortedDocValues termsIndex, long ord, BytesRef value) {
-        final int previousOrd = (int) (ord >> 2);
-        final int nextOrd = previousOrd + 1;
-        final BytesRef previous = previousOrd < 0 ? null : termsIndex.lookupOrd(previousOrd);
-        if ((ord & 3) == 0) { // there was an existing ord with the inserted value
-            assert compareValues(previous, value) == 0;
-        } else {
-            assert compareValues(previous, value) < 0;
-        }
-        if (nextOrd < termsIndex.getValueCount()) {
-            final BytesRef next = termsIndex.lookupOrd(nextOrd);
-            assert compareValues(value, next) < 0;
-        }
-        return true;
-    }
-
-    // find where to insert an ord in the current terms index
-    private long ordInCurrentReader(SortedDocValues termsIndex, BytesRef value) {
-        final long ord;
-        if (value == null) {
-            ord = -1 << 2;
-        } else {
-            final long docOrd = termsIndex.lookupTerm(value);
-            if (docOrd >= 0) {
-                // value exists in the current segment
-                ord = docOrd << 2;
-            } else {
-                // value doesn't exist, use the ord between the previous and the next term
-                ord = ((-2 - docOrd) << 2) + 2;
-            }
-        }
-
-        assert (ord & 1) == 0;
-        return ord;
-    }
-
-    @Override
-    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-        termsIndex = sortMode.select(indexFieldData.load(context).getOrdinalsValues(), -1);
-        missingOrd = ordInCurrentReader(termsIndex, missingValue);
-        assert consistentInsertedOrd(termsIndex, missingOrd, missingValue);
-        FieldComparator<BytesRef> perSegComp = new PerSegmentComparator(termsIndex);
-        currentReaderGen++;
-        if (bottomSlot != -1) {
-            perSegComp.setBottom(bottomSlot);
-        }
-        if (top != null) {
-            perSegComp.setTopValue(top);
-            topOrd = ordInCurrentReader(termsIndex, top);
-        } else {
-            topOrd = missingOrd;
-        }
-        return perSegComp;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        bottomSlot = bottom;
-        final BytesRef bottomValue = values[bottomSlot];
-
-        if (currentReaderGen == readerGen[bottomSlot]) {
-            bottomOrd = ords[bottomSlot];
-        } else {
-            // insert an ord
-            bottomOrd = ordInCurrentReader(termsIndex, bottomValue);
-            if (bottomOrd == missingOrd && bottomValue != null) {
-                // bottomValue and missingValue and in-between the same field data values -> tie-break
-                // this is why we multiply ords by 4
-                assert missingValue != null;
-                final int cmp = bottomValue.compareTo(missingValue);
-                if (cmp < 0) {
-                    --bottomOrd;
-                } else if (cmp > 0) {
-                    ++bottomOrd;
-                }
-            }
-            assert consistentInsertedOrd(termsIndex, bottomOrd, bottomValue);
-        }
-    }
-
-    @Override
-    public void setTopValue(BytesRef value) {
-        this.top = value;
-    }
-
-    @Override
-    public BytesRef value(int slot) {
-        return values[slot];
-    }
-
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java

@@ -1,136 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-
-/**
- * Sorts by field's natural Term sort order.  All
- * comparisons are done using BytesRef.compareTo, which is
- * slow for medium to large result sets but possibly
- * very fast for very small results sets.
- */
-public final class BytesRefValComparator extends NestedWrappableComparator<BytesRef> {
-
-    private final IndexFieldData<?> indexFieldData;
-    private final MultiValueMode sortMode;
-    private final BytesRef missingValue;
-
-    private final BytesRef[] values;
-    private BytesRef bottom;
-    private BytesRef top;
-    private BinaryDocValues docTerms;
-
-    BytesRefValComparator(IndexFieldData<?> indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) {
-        this.sortMode = sortMode;
-        values = new BytesRef[numHits];
-        this.indexFieldData = indexFieldData;
-        this.missingValue = missingValue;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final BytesRef val1 = values[slot1];
-        final BytesRef val2 = values[slot2];
-        return compareValues(val1, val2);
-    }
-
-    @Override
-    public int compareBottom(int doc) throws IOException {
-        BytesRef val2 = docTerms.get(doc);
-        return compareValues(bottom, val2);
-    }
-
-    @Override
-    public int compareTop(int doc) throws IOException {
-        return  top.compareTo(docTerms.get(doc));
-    }
-
-    @Override
-    public void copy(int slot, int doc) throws IOException {
-        BytesRef relevantValue = docTerms.get(doc);
-        if (relevantValue == missingValue) {
-            values[slot] = missingValue;
-        } else {
-            if (values[slot] == null || values[slot] == missingValue) {
-                values[slot] = new BytesRef();
-            }
-            values[slot].copyBytes(relevantValue);
-        }
-    }
-
-    @Override
-    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-        final SortedBinaryDocValues docTerms = indexFieldData.load(context).getBytesValues();
-        this.docTerms = sortMode.select(docTerms, missingValue);
-        return this;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public void setTopValue(BytesRef top) {
-        this.top = top;
-    }
-
-    @Override
-    public BytesRef value(int slot) {
-        return values[slot];
-    }
-
-    @Override
-    public int compareValues(BytesRef val1, BytesRef val2) {
-        if (val1 == null) {
-            if (val2 == null) {
-                return 0;
-            }
-            return -1;
-        } else if (val2 == null) {
-            return 1;
-        }
-        return val1.compareTo(val2);
-    }
-
-    @Override
-    public void missing(int slot) {
-        values[slot] = missingValue;
-    }
-
-    @Override
-    public int compareBottomMissing() {
-        return compareValues(bottom, missingValue);
-    }
-
-    @Override
-    public int compareTopMissing() {
-        return compareValues(top, missingValue);
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java

@@ -1,155 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.script.SearchScript;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Double
-public class DoubleScriptDataComparator extends NumberComparatorBase<Double> {
-
-    public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) {
-        return new InnerSource(script);
-    }
-
-    private static class InnerSource extends IndexFieldData.XFieldComparatorSource {
-
-        private final SearchScript script;
-
-        private InnerSource(SearchScript script) {
-            this.script = script;
-        }
-
-        @Override
-        public FieldComparator<? extends Number> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-            return new DoubleScriptDataComparator(numHits, script);
-        }
-
-        @Override
-        public SortField.Type reducedType() {
-            return SortField.Type.DOUBLE;
-        }
-    }
-
-    private final SearchScript script;
-
-    private final double[] values;
-    private double bottom;
-
-    public DoubleScriptDataComparator(int numHits, SearchScript script) {
-        this.script = script;
-        values = new double[numHits];
-    }
-
-    @Override
-    public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
-        script.setNextReader(context);
-        return this;
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) {
-        script.setScorer(scorer);
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final double v1 = values[slot1];
-        final double v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        script.setNextDocId(doc);
-        final double v2 = script.runAsDouble();
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareTop(int doc) throws IOException {
-        script.setNextDocId(doc);
-        double docValue = script.runAsDouble();
-        return Double.compare(top, docValue);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        script.setNextDocId(doc);
-        values[slot] = script.runAsDouble();
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Double value(int slot) {
-        return values[slot];
-    }
-
-    @Override
-    public void add(int slot, int doc) {
-        script.setNextDocId(doc);
-        values[slot] += script.runAsDouble();
-    }
-
-    @Override
-    public void divide(int slot, int divisor) {
-        values[slot] /= divisor;
-    }
-
-    @Override
-    public void missing(int slot) {
-        values[slot] = Double.MAX_VALUE;
-    }
-
-    @Override
-    public int compareBottomMissing() {
-        return Double.compare(bottom, Double.MAX_VALUE);
-    }
-
-    @Override
-    public int compareTopMissing() {
-        return Double.compare(top, Double.MAX_VALUE);
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java

@@ -1,74 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-
-/**
- */
-public class DoubleValuesComparator extends DoubleValuesComparatorBase<Double> {
-
-    private final double[] values;
-
-    public DoubleValuesComparator(IndexNumericFieldData indexFieldData, double missingValue, int numHits, MultiValueMode sortMode) {
-        super(indexFieldData, missingValue, sortMode);
-        this.values = new double[numHits];
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final double v1 = values[slot1];
-        final double v2 = values[slot2];
-        return compare(v1, v2);
-    }
-
-    @Override
-    public void setBottom(int slot) {
-        this.bottom = values[slot];
-    }
-
-    @Override
-    public void copy(int slot, int doc) throws IOException {
-        values[slot] = readerValues.get(doc);
-    }
-
-    @Override
-    public Double value(int slot) {
-        return Double.valueOf(values[slot]);
-    }
-
-    @Override
-    public void add(int slot, int doc) {
-        values[slot] += readerValues.get(doc);
-    }
-
-    @Override
-    public void divide(int slot, int divisor) {
-        values[slot] /= divisor;
-    }
-
-    @Override
-    public void missing(int slot) {
-        values[slot] = missingValue;
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorBase.java

@@ -1,79 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.FieldComparator;
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.index.fielddata.NumericDoubleValues;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-
-abstract class DoubleValuesComparatorBase<T extends Number> extends NumberComparatorBase<T> {
-
-    protected final IndexNumericFieldData indexFieldData;
-    protected final double missingValue;
-    protected double bottom;
-    protected NumericDoubleValues readerValues;
-    protected final MultiValueMode sortMode;
-
-    public DoubleValuesComparatorBase(IndexNumericFieldData indexFieldData, double missingValue, MultiValueMode sortMode) {
-        this.indexFieldData = indexFieldData;
-        this.missingValue = missingValue;
-        this.sortMode = sortMode;
-    }
-
-    @Override
-    public final int compareBottom(int doc) throws IOException {
-        final double v2 = readerValues.get(doc);
-        return compare(bottom, v2);
-    }
-
-    @Override
-    public int compareTop(int doc) throws IOException {
-        return compare(top.doubleValue(), readerValues.get(doc));
-    }
-
-    protected NumericDoubleValues getNumericDoubleValues(AtomicReaderContext context) {
-        SortedNumericDoubleValues readerValues = indexFieldData.load(context).getDoubleValues();
-        return sortMode.select(readerValues, missingValue);
-    }
-
-    @Override
-    public final FieldComparator<T> setNextReader(AtomicReaderContext context) throws IOException {
-        this.readerValues = getNumericDoubleValues(context);
-        return this;
-    }
-
-    @Override
-    public int compareBottomMissing() {
-        return compare(bottom, missingValue);
-    }
-
-    @Override
-    public int compareTopMissing() {
-        return compare(top.doubleValue(), missingValue);
-    }
-
-    static final int compare(double left, double right) {
-        return Double.compare(left, right);
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java

@@ -19,27 +19,36 @@
 
 package org.elasticsearch.index.fielddata.fieldcomparator;
 
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldCache.Doubles;
 import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
 /**
+ * Comparator source for double values.
  */
 public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
 
     private final IndexNumericFieldData indexFieldData;
     private final Object missingValue;
     private final MultiValueMode sortMode;
+    private final Nested nested;
 
-    public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
+    public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         this.indexFieldData = indexFieldData;
         this.missingValue = missingValue;
         this.sortMode = sortMode;
+        this.nested = nested;
     }
 
     @Override
@@ -47,11 +56,42 @@ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparato
         return SortField.Type.DOUBLE;
     }
 
+    protected SortedNumericDoubleValues getValues(AtomicReaderContext context) {
+        return indexFieldData.load(context).getDoubleValues();
+    }
+
+    protected void setScorer(Scorer scorer) {}
+
     @Override
     public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+        assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
 
         final double dMissingValue = (Double) missingObject(missingValue, reversed);
-        return new DoubleValuesComparator(indexFieldData, dMissingValue, numHits, sortMode);
+        // NOTE: it's important to pass null as a missing value in the constructor so that
+        // the comparator doesn't check docsWithField since we replace missing values in select()
+        return new FieldComparator.DoubleComparator(numHits, null, null, null) {
+            @Override
+            protected Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException {
+                final SortedNumericDoubleValues values = getValues(context);
+                final NumericDoubleValues selectedValues;
+                if (nested == null) {
+                    selectedValues = sortMode.select(values, dMissingValue);
+                } else {
+                    final FixedBitSet rootDocs = nested.rootDocs(context);
+                    final FixedBitSet innerDocs = nested.innerDocs(context);
+                    selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
+                }
+                return new Doubles() {
+                    @Override
+                    public double get(int docID) {
+                        return selectedValues.get(docID);
+                    }
+                };
+            }
+            @Override
+            public void setScorer(Scorer scorer) {
+                DoubleValuesComparatorSource.this.setScorer(scorer);
+            }
+        };
     }
 }

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java

@@ -1,74 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-
-/**
- */
-public final class FloatValuesComparator extends DoubleValuesComparatorBase<Float> {
-
-    private final float[] values;
-
-    public FloatValuesComparator(IndexNumericFieldData indexFieldData, float missingValue, int numHits, MultiValueMode sortMode) {
-        super(indexFieldData, missingValue, sortMode);
-        assert indexFieldData.getNumericType().requiredBits() <= 32;
-        this.values = new float[numHits];
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final float v1 = values[slot1];
-        final float v2 = values[slot2];
-        return Float.compare(v1, v2);
-    }
-
-    @Override
-    public void setBottom(int slot) {
-        this.bottom = values[slot];
-    }
-
-    @Override
-    public void copy(int slot, int doc) throws IOException {
-        values[slot] = (float) readerValues.get(doc);
-    }
-
-    @Override
-    public Float value(int slot) {
-        return Float.valueOf(values[slot]);
-    }
-
-    @Override
-    public void add(int slot, int doc) {
-        values[slot] += (float) readerValues.get(doc);
-    }
-
-    @Override
-    public void divide(int slot, int divisor) {
-        values[slot] /= divisor;
-    }
-
-    @Override
-    public void missing(int slot) {
-        values[slot] = (float) missingValue;
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java

@@ -18,27 +18,35 @@
  */
 package org.elasticsearch.index.fielddata.fieldcomparator;
 
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldCache.Floats;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
 /**
+ * Comparator source for float values.
  */
 public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
 
     private final IndexNumericFieldData indexFieldData;
     private final Object missingValue;
     private final MultiValueMode sortMode;
+    private final Nested nested;
 
-    public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
+    public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         this.indexFieldData = indexFieldData;
         this.missingValue = missingValue;
         this.sortMode = sortMode;
+        this.nested = nested;
     }
 
     @Override
@@ -48,9 +56,30 @@ public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparator
 
     @Override
     public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+        assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
 
         final float dMissingValue = (Float) missingObject(missingValue, reversed);
-        return new FloatValuesComparator(indexFieldData, dMissingValue, numHits, sortMode);
+        // NOTE: it's important to pass null as a missing value in the constructor so that
+        // the comparator doesn't check docsWithField since we replace missing values in select()
+        return new FieldComparator.FloatComparator(numHits, null, null, null) {
+            @Override
+            protected Floats getFloatValues(AtomicReaderContext context, String field) throws IOException {
+                final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues();
+                final NumericDoubleValues selectedValues;
+                if (nested == null) {
+                    selectedValues = sortMode.select(values, dMissingValue);
+                } else {
+                    final FixedBitSet rootDocs = nested.rootDocs(context);
+                    final FixedBitSet innerDocs = nested.innerDocs(context);
+                    selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
+                }
+                return new Floats() {
+                    @Override
+                    public float get(int docID) {
+                        return (float) selectedValues.get(docID);
+                    }
+                };
+            }
+        };
     }
 }

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java

@@ -1,73 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-
-/**
- */
-public final class LongValuesComparator extends LongValuesComparatorBase<Long> {
-
-    private final long[] values;
-
-    public LongValuesComparator(IndexNumericFieldData indexFieldData, long missingValue, int numHits, MultiValueMode sortMode) {
-        super(indexFieldData, missingValue, sortMode);
-        this.values = new long[numHits];
-        assert indexFieldData.getNumericType().requiredBits() <= 64;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final long v1 = values[slot1];
-        final long v2 = values[slot2];
-        return Long.compare(v1, v2);
-    }
-
-    @Override
-    public void setBottom(int slot) {
-        this.bottom = values[slot];
-    }
-
-    public void copy(int slot, int doc) throws IOException {
-        values[slot] = readerValues.get(doc);
-    }
-
-    @Override
-    public Long value(int slot) {
-        return Long.valueOf(values[slot]);
-    }
-
-    @Override
-    public void add(int slot, int doc) {
-        values[slot] += readerValues.get(doc);
-    }
-
-    @Override
-    public void divide(int slot, int divisor) {
-        values[slot] /= divisor;
-    }
-
-    @Override
-    public void missing(int slot) {
-        values[slot] = missingValue;
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorBase.java

@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.search.FieldComparator;
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-
-abstract class LongValuesComparatorBase<T extends Number> extends NumberComparatorBase<T> {
-
-    protected final IndexNumericFieldData indexFieldData;
-    protected final long missingValue;
-    protected long bottom;
-    protected NumericDocValues readerValues;
-    protected final MultiValueMode sortMode;
-
-
-    public LongValuesComparatorBase(IndexNumericFieldData indexFieldData, long missingValue, MultiValueMode sortMode) {
-        this.indexFieldData = indexFieldData;
-        this.missingValue = missingValue;
-        this.sortMode = sortMode;
-    }
-
-    @Override
-    public final int compareBottom(int doc) throws IOException {
-        long v2 = readerValues.get(doc);
-        return Long.compare(bottom, v2);
-    }
-
-    @Override
-    public int compareTop(int doc) throws IOException {
-        return Long.compare(top.longValue(), readerValues.get(doc));
-    }
-
-    @Override
-    public final FieldComparator<T> setNextReader(AtomicReaderContext context) throws IOException {
-        SortedNumericDocValues readerValues = indexFieldData.load(context).getLongValues();
-        this.readerValues = sortMode.select(readerValues, missingValue);
-        return this;
-    }
-
-    @Override
-    public int compareBottomMissing() {
-        return Long.compare(bottom, missingValue);
-    }
-
-    @Override
-    public int compareTopMissing() {
-        return Long.compare(top.longValue(), missingValue);
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java

@@ -18,8 +18,13 @@
  */
 package org.elasticsearch.index.fielddata.fieldcomparator;
 
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.FieldCache.Longs;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -28,17 +33,20 @@ import org.elasticsearch.search.MultiValueMode;
 import java.io.IOException;
 
 /**
+ * Comparator source for long values.
  */
 public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
 
     private final IndexNumericFieldData indexFieldData;
     private final Object missingValue;
     private final MultiValueMode sortMode;
+    private final Nested nested;
 
-    public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
+    public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         this.indexFieldData = indexFieldData;
         this.missingValue = missingValue;
         this.sortMode = sortMode;
+        this.nested = nested;
     }
 
     @Override
@@ -48,9 +56,31 @@ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorS
 
     @Override
     public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+        assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
 
-        final long dMissingValue = (Long) missingObject(missingValue, reversed);
+        final Long dMissingValue = (Long) missingObject(missingValue, reversed);
-        return new LongValuesComparator(indexFieldData, dMissingValue, numHits, sortMode);
+        // NOTE: it's important to pass null as a missing value in the constructor so that
+        // the comparator doesn't check docsWithField since we replace missing values in select()
+        return new FieldComparator.LongComparator(numHits, null, null, null) {
+            @Override
+            protected Longs getLongValues(AtomicReaderContext context, String field) throws IOException {
+                final SortedNumericDocValues values = indexFieldData.load(context).getLongValues();
+                final NumericDocValues selectedValues;
+                if (nested == null) {
+                    selectedValues = sortMode.select(values, dMissingValue);
+                } else {
+                    final FixedBitSet rootDocs = nested.rootDocs(context);
+                    final FixedBitSet innerDocs = nested.innerDocs(context);
+                    selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
+                }
+                return new Longs() {
+                    @Override
+                    public long get(int docID) {
+                        return selectedValues.get(docID);
+                    }
+                };
+            }
+
+        };
     }
 }

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java

@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.search.FieldComparator;
-
-/** Base comparator which allows for nested sorting. */
-public abstract class NestedWrappableComparator<T> extends FieldComparator<T> {
-
-    /**
-     * Assigns the underlying missing value to the specified slot, if the actual implementation supports missing value.
-     *
-     * @param slot The slot to assign the the missing value to.
-     */
-    public abstract void missing(int slot);
-
-    /**
-     * Compares the missing value to the bottom.
-     *
-     * @return any N < 0 if the bottom value is not competitive with the missing value, any N > 0 if the
-     * bottom value is competitive with the missing value and 0 if they are equal.
-     */
-    public abstract int compareBottomMissing();
-
-    /**
-     * Compares the missing value to the top.
-     *
-     * @return any N < 0 if the tope is not competitive with the missing value, any N > 0 if the top is competitive
-     * with the top and 0 if they are equal.
-     */
-    public abstract int compareTopMissing();
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java

@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-
-/**
- * Base FieldComparator class for number fields.
- */
-// This is right now only used for sorting number based fields inside nested objects
-public abstract class NumberComparatorBase<T> extends NestedWrappableComparator<T> {
-
-    protected T top;
-    /**
-     * Adds numeric value at the specified doc to the specified slot.
-     *
-     * @param slot  The specified slot
-     * @param doc   The specified doc
-     */
-    public abstract void add(int slot, int doc);
-
-    /**
-     * Divides the value at the specified slot with the specified divisor.
-     *
-     * @param slot      The specified slot
-     * @param divisor   The specified divisor
-     */
-    public abstract void divide(int slot, int divisor);
-
-    @Override
-    public void setTopValue(T top) {
-        this.top = top;
-    }
-
-}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java

@@ -1,154 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.script.SearchScript;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class StringScriptDataComparator extends FieldComparator<BytesRef> {
-
-    private BytesRef top;
-
-    public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) {
-        return new InnerSource(script);
-    }
-
-    private static class InnerSource extends IndexFieldData.XFieldComparatorSource {
-
-        private final SearchScript script;
-
-        private InnerSource(SearchScript script) {
-            this.script = script;
-        }
-
-        @Override
-        public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-            return new StringScriptDataComparator(numHits, script);
-        }
-
-        @Override
-        public SortField.Type reducedType() {
-            return SortField.Type.STRING;
-        }
-    }
-
-    private final SearchScript script;
-
-    private BytesRef[] values; // TODO maybe we can preallocate or use a sentinel to prevent the conditionals in compare
-
-    private BytesRef bottom;
-    
-    private final BytesRef spare = new BytesRef();
-    
-    private int spareDoc = -1;
-
-    public StringScriptDataComparator(int numHits, SearchScript script) {
-        this.script = script;
-        values = new BytesRef[numHits];
-    }
-
-    @Override
-    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-        script.setNextReader(context);
-        spareDoc = -1;
-        return this;
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) {
-        script.setScorer(scorer);
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final BytesRef val1 = values[slot1];
-        final BytesRef val2 = values[slot2];
-        if (val1 == null) {
-            if (val2 == null) {
-                return 0;
-            }
-            return -1;
-        } else if (val2 == null) {
-            return 1;
-        }
-
-        return val1.compareTo(val2);
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        if (bottom == null) {
-            return -1;
-        }
-        setSpare(doc);
-        return bottom.compareTo(spare);
-    }
-
-    @Override
-    public int compareTop(int doc) throws IOException {
-        script.setNextDocId(doc);
-        setSpare(doc);
-        return top.compareTo(spare);
-    }
-
-    private void setSpare(int doc) {
-        if (spareDoc == doc) {
-            return;
-        }
-        script.setNextDocId(doc);
-        spare.copyChars(script.run().toString());
-        spareDoc = doc;
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        setSpare(doc);
-        if (values[slot] == null) {
-           values[slot] = BytesRef.deepCopyOf(spare);
-        } else {
-            values[slot].copyBytes(spare);
-        }
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public void setTopValue(BytesRef top) {
-        this.top = top;
-    }
-
-    @Override
-    public BytesRef value(int slot) {
-        return values[slot];
-    }
-}

src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java

@@ -25,10 +25,8 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.search.MultiValueMode;
 
@@ -74,7 +72,7 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         throw new UnsupportedOperationException("no global ordinals sorting yet");
     }
 

src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java

@@ -28,10 +28,8 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexFieldDataCache;
-import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
 import org.elasticsearch.search.MultiValueMode;
 
@@ -80,7 +78,7 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData<Ato
     }
 
     @Override
-    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         throw new ElasticsearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
     }
 

src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java

@@ -26,10 +26,8 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexFieldDataCache;
-import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
@@ -57,8 +55,8 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
+        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
     }
 
     @Override

src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java

@@ -23,6 +23,7 @@ import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
 import org.elasticsearch.search.MultiValueMode;
@@ -44,7 +45,7 @@ public class BinaryDVIndexFieldData extends DocValuesIndexFieldData implements I
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
+    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
+        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
     }
 }

src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java

@@ -31,10 +31,8 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
 import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
 import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
@@ -53,15 +51,15 @@ public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData imple
         this.numericType = numericType;
     }
 
-    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(final Object missingValue, final MultiValueMode sortMode) {
+    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(final Object missingValue, final MultiValueMode sortMode, Nested nested) {
         switch (numericType) {
         case FLOAT:
-            return new FloatValuesComparatorSource(this, missingValue, sortMode);
+            return new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
         case DOUBLE:
-            return new DoubleValuesComparatorSource(this, missingValue, sortMode);
+            return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
         default:
             assert !numericType.isFloatingPoint();
-            return new LongValuesComparatorSource(this, missingValue, sortMode);
+            return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
         }
     }
 

src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java

@@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
@@ -44,7 +45,7 @@ public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData impleme
     }
 
     @Override
-    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         throw new ElasticsearchIllegalArgumentException("can't sort on binary field");
     }
 

src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java

@@ -23,10 +23,8 @@ import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
 import org.elasticsearch.index.mapper.MapperService;
@@ -59,7 +57,7 @@ public final class DisabledIndexFieldData extends AbstractIndexFieldData<AtomicF
     }
 
     @Override
-    public IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
+    public IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
         throw fail();
     }
 

src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java

@@ -27,6 +27,7 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@@ -155,8 +156,8 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new DoubleValuesComparatorSource(this, missingValue, sortMode);
+        return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
     }
 
     private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final DoubleArray values, int maxDoc) {

src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.FloatArray;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@@ -153,8 +154,8 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<AtomicNumer
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new FloatValuesComparatorSource(this, missingValue, sortMode);
+        return new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
     }
 
     private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final FloatArray values, int maxDoc) {

src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java

@@ -27,6 +27,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
 import org.elasticsearch.index.mapper.MapperService;
@@ -42,7 +43,7 @@ public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData impl
     }
 
     @Override
-    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         throw new ElasticsearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
     }
 

src/main/java/org/elasticsearch/index/fielddata/plain/NumericDVIndexFieldData.java

@@ -24,6 +24,7 @@ import org.apache.lucene.util.Bits;
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
@@ -62,8 +63,8 @@ public class NumericDVIndexFieldData extends DocValuesIndexFieldData implements
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
+    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new LongValuesComparatorSource(this, missingValue, sortMode);
+        return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
     }
 
     @Override

src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java

@@ -31,6 +31,7 @@ import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@@ -343,8 +344,8 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new LongValuesComparatorSource(this, missingValue, sortMode);
+        return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
     }
 
     /**

src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildAtomicFieldData.java

@@ -63,7 +63,7 @@ public class ParentChildAtomicFieldData extends AbstractAtomicParentChildFieldDa
     public SortedDocValues getOrdinalsValues(String type) {
         AtomicOrdinalsFieldData atomicFieldData = typeToIds.get(type);
         if (atomicFieldData != null) {
-            return MultiValueMode.MIN.select(atomicFieldData.getOrdinalsValues(), -1);
+            return MultiValueMode.MIN.select(atomicFieldData.getOrdinalsValues());
         } else {
             return DocValues.emptySorted();
         }

src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java

@@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@@ -79,8 +80,8 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
     }
 
     @Override
-    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
+        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
     }
 
     @Override
@@ -408,8 +409,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
         }
 
         @Override
-        public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue,
+        public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-                MultiValueMode sortMode) {
             throw new UnsupportedOperationException("No sorting on global ords");
         }
 

src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java

@@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
@@ -44,8 +45,8 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
         this.breakerService = breakerService;
     }
 
-    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
+    public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-        return new BytesRefFieldComparatorSource((IndexFieldData<?>) this, missingValue, sortMode);
+        return new BytesRefFieldComparatorSource((IndexFieldData<?>) this, missingValue, sortMode, nested);
     }
 
     @Override

src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java

@@ -1,500 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.search.nested;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.util.FixedBitSet;
-import org.elasticsearch.ElasticsearchIllegalArgumentException;
-import org.elasticsearch.common.lucene.docset.DocIdSets;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.fieldcomparator.NestedWrappableComparator;
-import org.elasticsearch.index.fielddata.fieldcomparator.NumberComparatorBase;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.io.IOException;
-import java.util.Locale;
-
-/**
- */
-public class NestedFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
-
-    private final MultiValueMode sortMode;
-    private final IndexFieldData.XFieldComparatorSource wrappedSource;
-    private final Filter rootDocumentsFilter;
-    private final Filter innerDocumentsFilter;
-
-    public NestedFieldComparatorSource(MultiValueMode sortMode, IndexFieldData.XFieldComparatorSource wrappedSource, Filter rootDocumentsFilter, Filter innerDocumentsFilter) {
-        this.sortMode = sortMode;
-        this.wrappedSource = wrappedSource;
-        this.rootDocumentsFilter = rootDocumentsFilter;
-        this.innerDocumentsFilter = innerDocumentsFilter;
-    }
-
-    @Override
-    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-        // +1: have one spare slot for value comparison between inner documents.
-        FieldComparator wrappedComparator = wrappedSource.newComparator(fieldname, numHits + 1, sortPos, reversed);
-        switch (sortMode) {
-            case MAX:
-                return new NestedFieldComparator.Highest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
-            case MIN:
-                return new NestedFieldComparator.Lowest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
-            case SUM:
-                return new NestedFieldComparator.Sum((NumberComparatorBase<?>) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
-            case AVG:
-                return new NestedFieldComparator.Avg((NumberComparatorBase<?>) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
-            default:
-                throw new ElasticsearchIllegalArgumentException(
-                    String.format(Locale.ROOT, "Unsupported sort_mode[%s] for nested type", sortMode)
-                );
-        }
-    }
-
-    @Override
-    public SortField.Type reducedType() {
-        return wrappedSource.reducedType();
-    }
-
-}
-
-abstract class NestedFieldComparator extends FieldComparator {
-
-    final Filter rootDocumentsFilter;
-    final Filter innerDocumentsFilter;
-    final int spareSlot;
-
-    FieldComparator wrappedComparator;
-    FixedBitSet rootDocuments;
-    FixedBitSet innerDocuments;
-    int bottomSlot;
-    Object top;
-
-    NestedFieldComparator(FieldComparator wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
-        this.wrappedComparator = wrappedComparator;
-        this.rootDocumentsFilter = rootDocumentsFilter;
-        this.innerDocumentsFilter = innerDocumentsFilter;
-        this.spareSlot = spareSlot;
-    }
-
-    @Override
-    public final int compare(int slot1, int slot2) {
-        return wrappedComparator.compare(slot1, slot2);
-    }
-
-    @Override
-    public final void setBottom(int slot) {
-        wrappedComparator.setBottom(slot);
-        this.bottomSlot = slot;
-    }
-
-    @Override
-    public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
-        DocIdSet innerDocuments = innerDocumentsFilter.getDocIdSet(context, null);
-        if (DocIdSets.isEmpty(innerDocuments)) {
-            this.innerDocuments = null;
-        } else if (innerDocuments instanceof FixedBitSet) {
-            this.innerDocuments = (FixedBitSet) innerDocuments;
-        } else {
-            this.innerDocuments = DocIdSets.toFixedBitSet(innerDocuments.iterator(), context.reader().maxDoc());
-        }
-        DocIdSet rootDocuments = rootDocumentsFilter.getDocIdSet(context, null);
-        if (DocIdSets.isEmpty(rootDocuments)) {
-            this.rootDocuments = null;
-        } else if (rootDocuments instanceof FixedBitSet) {
-            this.rootDocuments = (FixedBitSet) rootDocuments;
-        } else {
-            this.rootDocuments = DocIdSets.toFixedBitSet(rootDocuments.iterator(), context.reader().maxDoc());
-        }
-
-        wrappedComparator = wrappedComparator.setNextReader(context);
-        return this;
-    }
-
-    @Override
-    public final Object value(int slot) {
-        return wrappedComparator.value(slot);
-    }
-
-    @Override
-    public void setTopValue(Object top) {
-        this.top = top;
-        wrappedComparator.setTopValue(top);
-    }
-
-    final static class Lowest extends NestedFieldComparator {
-
-        Lowest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) {
-            super(wrappedComparator, parentFilter, childFilter, spareSlot);
-        }
-
-        @Override
-        public int compareBottom(int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                return compareBottomMissing(wrappedComparator);
-            }
-
-            // We need to copy the lowest value from all nested docs into slot.
-            int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                return compareBottomMissing(wrappedComparator);
-            }
-
-            // We only need to emit a single cmp value for any matching nested doc
-            int cmp = wrappedComparator.compareBottom(nestedDoc);
-            if (cmp > 0) {
-                return cmp;
-            }
-
-            while (true) {
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                    return cmp;
-                }
-                int cmp1 = wrappedComparator.compareBottom(nestedDoc);
-                if (cmp1 > 0) {
-                    return cmp1;
-                } else {
-                    if (cmp1 == 0) {
-                        cmp = 0;
-                    }
-                }
-            }
-        }
-
-        @Override
-        public void copy(int slot, int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                copyMissing(wrappedComparator, slot);
-                return;
-            }
-
-            // We need to copy the lowest value from all nested docs into slot.
-            int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                copyMissing(wrappedComparator, slot);
-                return;
-            }
-            wrappedComparator.copy(slot, nestedDoc);
-
-            while (true) {
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                    return;
-                }
-                wrappedComparator.copy(spareSlot, nestedDoc);
-                if (wrappedComparator.compare(spareSlot, slot) < 0) {
-                    wrappedComparator.copy(slot, nestedDoc);
-                }
-            }
-        }
-
-        @Override
-        public int compareTop(int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                return compareTopMissing(wrappedComparator);
-            }
-
-            // We need to copy the lowest value from all nested docs into slot.
-            int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                return compareTopMissing(wrappedComparator);
-            }
-
-            // We only need to emit a single cmp value for any matching nested doc
-            @SuppressWarnings("unchecked")
-            int cmp = wrappedComparator.compareTop(nestedDoc);
-            if (cmp > 0) {
-                return cmp;
-            }
-
-            while (true) {
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                    return cmp;
-                }
-                @SuppressWarnings("unchecked")
-                int cmp1 = wrappedComparator.compareTop(nestedDoc);
-                if (cmp1 > 0) {
-                    return cmp1;
-                } else {
-                    if (cmp1 == 0) {
-                        cmp = 0;
-                    }
-                }
-            }
-        }
-    }
-
-    final static class Highest extends NestedFieldComparator {
-
-        Highest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) {
-            super(wrappedComparator, parentFilter, childFilter, spareSlot);
-        }
-
-        @Override
-        public int compareBottom(int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                return compareBottomMissing(wrappedComparator);
-            }
-
-            int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                return compareBottomMissing(wrappedComparator);
-            }
-
-            int cmp = wrappedComparator.compareBottom(nestedDoc);
-            if (cmp < 0) {
-                return cmp;
-            }
-
-            while (true) {
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                    return cmp;
-                }
-                int cmp1 = wrappedComparator.compareBottom(nestedDoc);
-                if (cmp1 < 0) {
-                    return cmp1;
-                } else if (cmp1 == 0) {
-                   cmp = 0;
-                }
-            }
-        }
-
-        @Override
-        public void copy(int slot, int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                copyMissing(wrappedComparator, slot);
-                return;
-            }
-
-            int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                copyMissing(wrappedComparator, slot);
-                return;
-            }
-            wrappedComparator.copy(slot, nestedDoc);
-
-            while (true) {
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                    return;
-                }
-                wrappedComparator.copy(spareSlot, nestedDoc);
-                if (wrappedComparator.compare(spareSlot, slot) > 0) {
-                    wrappedComparator.copy(slot, nestedDoc);
-                }
-            }
-        }
-
-        @Override
-        public int compareTop(int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                return compareTopMissing(wrappedComparator);
-            }
-
-            int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                return compareTopMissing(wrappedComparator);
-            }
-
-            @SuppressWarnings("unchecked")
-            int cmp = wrappedComparator.compareTop(nestedDoc);
-            if (cmp < 0) {
-                return cmp;
-            }
-
-            while (true) {
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                    return cmp;
-                }
-                @SuppressWarnings("unchecked")
-                int cmp1 = wrappedComparator.compareTop(nestedDoc);
-                if (cmp1 < 0) {
-                    return cmp1;
-                } else if (cmp1 == 0) {
-                    cmp = 0;
-                }
-            }
-        }
-    }
-    
-    static abstract class NumericNestedFieldComparatorBase extends NestedFieldComparator {
-        protected NumberComparatorBase numberComparator;
-
-        NumericNestedFieldComparatorBase(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
-            super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot);
-            this.numberComparator = wrappedComparator;
-        }
-
-        @Override
-        public final int compareBottom(int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                return compareBottomMissing(wrappedComparator);
-            }
-
-            final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                return compareBottomMissing(wrappedComparator);
-            }
-
-            int counter = 1;
-            wrappedComparator.copy(spareSlot, nestedDoc);
-            nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-            while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) {
-                onNested(spareSlot, nestedDoc);
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                counter++;
-            }
-            afterNested(spareSlot, counter);
-            return compare(bottomSlot, spareSlot);
-        }
-
-        @Override
-        public final void copy(int slot, int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                copyMissing(wrappedComparator, slot);
-                return;
-            }
-
-            final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                copyMissing(wrappedComparator, slot);
-                return;
-            }
-            int counter = 1;
-            wrappedComparator.copy(slot, nestedDoc);
-            nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-            while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) {
-                onNested(slot, nestedDoc);
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                counter++;
-            }
-            afterNested(slot, counter);
-        }
-
-        @Override
-        @SuppressWarnings("unchecked")
-        public int compareTop(int rootDoc) throws IOException {
-            if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
-                return compareTopMissing(wrappedComparator);
-            }
-
-            final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
-            int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
-            if (nestedDoc >= rootDoc || nestedDoc == -1) {
-                return compareTopMissing(wrappedComparator);
-            }
-
-            int counter = 1;
-            wrappedComparator.copy(spareSlot, nestedDoc);
-            nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-            while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) {
-                onNested(spareSlot, nestedDoc);
-                nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
-                counter++;
-            }
-            afterNested(spareSlot, counter);
-            return wrappedComparator.compareValues(wrappedComparator.value(spareSlot), top);
-        }
-
-        protected abstract void onNested(int slot, int nestedDoc);
-        
-        protected abstract void afterNested(int slot, int count);
-
-        @Override
-        public final FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
-            super.setNextReader(context);
-            numberComparator = (NumberComparatorBase) super.wrappedComparator;
-            return this;
-        }
-    }
-
-    final static class Sum extends NumericNestedFieldComparatorBase {
-
-        Sum(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
-            super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot);
-        }
-
-        @Override
-        protected void onNested(int slot, int nestedDoc) {
-            numberComparator.add(slot, nestedDoc);
-        }
-
-        @Override
-        protected void afterNested(int slot, int count) {
-        }
-
-    }
-
-    final static class Avg extends NumericNestedFieldComparatorBase {
-        Avg(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
-            super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot);
-        }
-
-        @Override
-        protected void onNested(int slot, int nestedDoc) {
-            numberComparator.add(slot, nestedDoc);
-        }
-
-        @Override
-        protected void afterNested(int slot, int count) {
-            numberComparator.divide(slot, count);
-        }
-
-      
-    }
-
-    static final void copyMissing(FieldComparator<?> comparator, int slot) {
-        if (comparator instanceof NestedWrappableComparator<?>) {
-            ((NestedWrappableComparator<?>) comparator).missing(slot);
-        }
-    }
-
-    static final int compareBottomMissing(FieldComparator<?> comparator) {
-        if (comparator instanceof NestedWrappableComparator<?>) {
-            return ((NestedWrappableComparator<?>) comparator).compareBottomMissing();
-        } else {
-            return 0;
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    static final int compareTopMissing(FieldComparator<?> comparator) {
-        if (comparator instanceof NestedWrappableComparator) {
-            return ((NestedWrappableComparator) comparator).compareTopMissing();
-        } else {
-            return 0;
-        }
-    }
-
-}

src/main/java/org/elasticsearch/search/MultiValueMode.java

@@ -23,6 +23,7 @@ package org.elasticsearch.search;
 import org.apache.lucene.index.*;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
@@ -91,7 +92,13 @@ public enum MultiValueMode {
          */
         @Override
         public long reduce(long a, int numValues) {
-            return Math.round(a / Math.max(1.0, numValues));
+            if (numValues <= 1) {
+                // without this the average might be different from the original
+                // values on a single-valued field due to the precision loss of the double
+                return a;
+            } else {
+                return Math.round(a / Math.max(1.0, numValues));
+            }
         }
     },
 
@@ -115,6 +122,16 @@ public enum MultiValueMode {
             return Math.min(a, b);
         }
 
+        @Override
+        public int applyOrd(int ord1, int ord2) {
+            return Math.min(ord1, ord2);
+        }
+
+        @Override
+        public BytesRef apply(BytesRef a, BytesRef b) {
+            return a.compareTo(b) <= 0 ? a : b;
+        }
+
         /**
          * Returns {@link Double#POSITIVE_INFINITY}
          */
@@ -159,11 +176,11 @@ public enum MultiValueMode {
         }
 
         @Override
-        protected int pick(RandomAccessOrds values, int missingOrd) {
+        protected int pick(RandomAccessOrds values) {
             if (values.cardinality() > 0) {
                 return (int) values.ordAt(0);
             } else {
-                return missingOrd;
+                return -1;
             }
         }
     },
@@ -188,6 +205,16 @@ public enum MultiValueMode {
             return Math.max(a, b);
         }
 
+        @Override
+        public int applyOrd(int ord1, int ord2) {
+            return Math.max(ord1, ord2);
+        }
+
+        @Override
+        public BytesRef apply(BytesRef a, BytesRef b) {
+            return a.compareTo(b) > 0 ? a : b;
+        }
+
         /**
          * Returns {@link Double#NEGATIVE_INFINITY}
          */
@@ -236,12 +263,12 @@ public enum MultiValueMode {
         }
 
         @Override
-        protected int pick(RandomAccessOrds values, int missingOrd) {
+        protected int pick(RandomAccessOrds values) {
             final int count = values.cardinality();
             if (count > 0) {
                 return (int) values.ordAt(count - 1);
             } else {
-                return missingOrd;
+                return -1;
             }
         }
     };
@@ -274,6 +301,14 @@ public enum MultiValueMode {
      */
     public abstract long apply(long a, long b);
 
+    public int applyOrd(int ord1, int ord2) {
+        throw new UnsupportedOperationException();
+    }
+
+    public BytesRef apply(BytesRef a, BytesRef b) {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * Returns an initial value for the sort mode that is guaranteed to have no impact if passed
      * to {@link #apply(double, double)}. This value should be used as the initial value if the
@@ -359,6 +394,11 @@ public enum MultiValueMode {
         }
     }
 
+    /**
+     * Return a {@link NumericDocValues} instance that can be used to sort documents
+     * with this mode and the provided values. When a document has no value,
+     * <code>missingValue</code> is returned.
+     */
     public NumericDocValues select(final SortedNumericDocValues values, final long missingValue) {
         final NumericDocValues singleton = DocValues.unwrapSingleton(values);
         if (singleton != null) {
@@ -388,6 +428,51 @@ public enum MultiValueMode {
         }
     }
 
+    /**
+     * Return a {@link NumericDocValues} instance that can be used to sort root documents
+     * with this mode, the provided values and filters for root/inner documents.
+     *
+     * For every root document, the values of its inner documents will be aggregated.
+     * If none of the inner documents has a value, then <code>missingValue</code> is returned.
+     *
+     * NOTE: Calling the returned instance on docs that are not root docs is illegal
+     */
+    public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) {
+        if (rootDocs == null || innerDocs == null) {
+            return select(FieldData.emptySortedNumeric(maxDoc), missingValue);
+        }
+        return new NumericDocValues() {
+
+            @Override
+            public long get(int rootDoc) {
+                assert rootDocs.get(rootDoc) : "can only sort root documents";
+                if (rootDoc == 0) {
+                    return missingValue;
+                }
+
+                final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
+                final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
+
+                long accumulated = startLong();
+                int numValues = 0;
+
+                for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
+                    values.setDocument(doc);
+                    final int count = values.count();
+                    for (int i = 0; i < count; ++i) {
+                        final long value = values.valueAt(i);
+                        accumulated = apply(accumulated, value);
+                    }
+                    numValues += count;
+                }
+
+                return numValues == 0
+                        ? missingValue
+                        : reduce(accumulated, numValues);
+            }
+        };
+    }
+
     protected double pick(SortedNumericDoubleValues values, double missingValue) {
         final int count = values.count();
         if (count == 0) {
@@ -401,6 +486,11 @@ public enum MultiValueMode {
         }
     }
 
+    /**
+     * Return a {@link NumericDoubleValues} instance that can be used to sort documents
+     * with this mode and the provided values. When a document has no value,
+     * <code>missingValue</code> is returned.
+     */
     public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue) {
         final NumericDoubleValues singleton = FieldData.unwrapSingleton(values);
         if (singleton != null) {
@@ -430,15 +520,65 @@ public enum MultiValueMode {
         }
     }
 
+    /**
+     * Return a {@link NumericDoubleValues} instance that can be used to sort root documents
+     * with this mode, the provided values and filters for root/inner documents.
+     *
+     * For every root document, the values of its inner documents will be aggregated.
+     * If none of the inner documents has a value, then <code>missingValue</code> is returned.
+     *
+     * NOTE: Calling the returned instance on docs that are not root docs is illegal
+     */
+    public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) {
+        if (rootDocs == null || innerDocs == null) {
+            return select(FieldData.emptySortedNumericDoubles(maxDoc), missingValue);
+        }
+        return new NumericDoubleValues() {
+
+            @Override
+            public double get(int rootDoc) {
+                assert rootDocs.get(rootDoc) : "can only sort root documents";
+                if (rootDoc == 0) {
+                    return missingValue;
+                }
+
+                final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
+                final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
+
+                double accumulated = startDouble();
+                int numValues = 0;
+
+                for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
+                    values.setDocument(doc);
+                    final int count = values.count();
+                    for (int i = 0; i < count; ++i) {
+                        final double value = values.valueAt(i);
+                        accumulated = apply(accumulated, value);
+                    }
+                    numValues += count;
+                }
+
+                return numValues == 0
+                        ? missingValue
+                        : reduce(accumulated, numValues);
+            }
+        };
+    }
+
     protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue) {
         throw new ElasticsearchIllegalArgumentException("Unsupported sort mode: " + this);
     }
 
+    /**
+     * Return a {@link BinaryDocValues} instance that can be used to sort documents
+     * with this mode and the provided values. When a document has no value,
+     * <code>missingValue</code> is returned.
+     */
     public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue) {
         final BinaryDocValues singleton = FieldData.unwrapSingleton(values);
         if (singleton != null) {
             final Bits docsWithField = FieldData.unwrapSingletonBits(values);
-            if (docsWithField == null || missingValue == null || missingValue.length == 0) {
+            if (docsWithField == null) {
                 return singleton;
             } else {
                 return new BinaryDocValues() {
@@ -463,47 +603,85 @@ public enum MultiValueMode {
         }
     }
 
-    protected int pick(RandomAccessOrds values, int missingOrd) {
+    /**
+     * Return a {@link BinaryDocValues} instance that can be used to sort root documents
+     * with this mode, the provided values and filters for root/inner documents.
+     *
+     * For every root document, the values of its inner documents will be aggregated.
+     * If none of the inner documents has a value, then <code>missingValue</code> is returned.
+     *
+     * NOTE: Calling the returned instance on docs that are not root docs is illegal
+     */
+    public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) {
+        if (rootDocs == null || innerDocs == null) {
+            return select(FieldData.emptySortedBinary(maxDoc), missingValue);
+        }
+        final BinaryDocValues selectedValues = select(values, new BytesRef());
+        final Bits docsWithValue;
+        if (FieldData.unwrapSingleton(values) != null) {
+            docsWithValue = FieldData.unwrapSingletonBits(values);
+        } else {
+            docsWithValue = FieldData.docsWithValue(values, maxDoc);
+        }
+        return new BinaryDocValues() {
+
+            final BytesRef spare = new BytesRef();
+
+            @Override
+            public BytesRef get(int rootDoc) {
+                assert rootDocs.get(rootDoc) : "can only sort root documents";
+                if (rootDoc == 0) {
+                    return missingValue;
+                }
+
+                final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
+                final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
+
+                BytesRef accumulated = null;
+
+                for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
+                    values.setDocument(doc);
+                    final BytesRef innerValue = selectedValues.get(doc);
+                    if (innerValue.length > 0 || docsWithValue == null || docsWithValue.get(doc)) {
+                        if (accumulated == null) {
+                            spare.copyBytes(innerValue);
+                            accumulated = spare;
+                        } else {
+                            final BytesRef applied = apply(accumulated, innerValue);
+                            if (applied == innerValue) {
+                                accumulated.copyBytes(innerValue);
+                            }
+                        }
+                    }
+                }
+
+                return accumulated == null ? missingValue : accumulated;
+            }
+        };
+    }
+
+    protected int pick(RandomAccessOrds values) {
         throw new ElasticsearchIllegalArgumentException("Unsupported sort mode: " + this);
     }
 
-    public SortedDocValues select(final RandomAccessOrds values, final int missingOrd) {
+    /**
+     * Return a {@link SortedDocValues} instance that can be used to sort documents
+     * with this mode and the provided values.
+     */
+    public SortedDocValues select(final RandomAccessOrds values) {
         if (values.getValueCount() >= Integer.MAX_VALUE) {
             throw new UnsupportedOperationException("fields containing more than " + (Integer.MAX_VALUE-1) + " unique terms are unsupported");
         }
 
         final SortedDocValues singleton = DocValues.unwrapSingleton(values);
         if (singleton != null) {
-            if (missingOrd == -1) {
+            return singleton;
-                return singleton;
-            } else {
-                return new SortedDocValues() {
-                    @Override
-                    public int getOrd(int docID) {
-                        final int ord = singleton.getOrd(docID);
-                        if (ord == -1) {
-                            return missingOrd;
-                        }
-                        return ord;
-                    }
-
-                    @Override
-                    public BytesRef lookupOrd(int ord) {
-                        return values.lookupOrd(ord);
-                    }
-
-                    @Override
-                    public int getValueCount() {
-                        return (int) values.getValueCount();
-                    }
-                };
-            }
         } else {
             return new SortedDocValues() {
                 @Override
                 public int getOrd(int docID) {
                     values.setDocument(docID);
-                    return pick(values, missingOrd);
+                    return pick(values);
                 }
 
                 @Override
@@ -519,4 +697,55 @@ public enum MultiValueMode {
         }
     }
 
+    /**
+     * Return a {@link SortedDocValues} instance that can be used to sort root documents
+     * with this mode, the provided values and filters for root/inner documents.
+     *
+     * For every root document, the values of its inner documents will be aggregated.
+     *
+     * NOTE: Calling the returned instance on docs that are not root docs is illegal
+     */
+    public SortedDocValues select(final RandomAccessOrds values, final FixedBitSet rootDocs, final FixedBitSet innerDocs) {
+        if (rootDocs == null || innerDocs == null) {
+            return select((RandomAccessOrds) DocValues.emptySortedSet());
+        }
+        final SortedDocValues selectedValues = select(values);
+        return new SortedDocValues() {
+
+            @Override
+            public BytesRef lookupOrd(int ord) {
+                return selectedValues.lookupOrd(ord);
+            }
+
+            @Override
+            public int getValueCount() {
+                return selectedValues.getValueCount();
+            }
+
+            @Override
+            public int getOrd(int rootDoc) {
+                assert rootDocs.get(rootDoc) : "can only sort root documents";
+                if (rootDoc == 0) {
+                    return -1;
+                }
+
+                final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
+                final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
+                int ord = -1;
+
+                for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
+                    final int innerOrd = selectedValues.getOrd(doc);
+                    if (innerOrd != -1) {
+                        if (ord == -1) {
+                            ord = innerOrd;
+                        } else {
+                            ord = applyOrd(ord, innerOrd);
+                        }
+                    }
+                }
+
+                return ord;
+            }
+        };
+    }
 }

src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java

@@ -20,10 +20,11 @@
 package org.elasticsearch.search.sort;
 
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldCache.Doubles;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortField.Type;
+import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
@@ -32,12 +33,11 @@ import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparator;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.ObjectMappers;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.index.query.ParsedFilter;
-import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
 import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.internal.SearchContext;
@@ -127,27 +127,6 @@ public class GeoDistanceSortParser implements SortParser {
         final MultiValueMode finalSortMode = sortMode; // final reference for use in the anonymous class
         final IndexGeoPointFieldData geoIndexFieldData = context.fieldData().getForField(mapper);
         final FixedSourceDistance distance = geoDistance.fixedSourceDistance(point.lat(), point.lon(), unit);
-        IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
-
-            @Override
-            public Type reducedType() {
-                return Type.DOUBLE;
-            }
-
-            @Override
-            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new DoubleValuesComparator(null, Double.MAX_VALUE, numHits, finalSortMode) {
-                    @Override
-                    protected NumericDoubleValues getNumericDoubleValues(AtomicReaderContext context) {
-                        final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
-                        final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(distance, geoPointValues);
-                        return sortMode.select(distanceValues, Double.MAX_VALUE);
-                    }
-
-                };
-            }
-
-        };
         ObjectMapper objectMapper;
         if (nestedPath != null) {
             ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath);
@@ -161,6 +140,7 @@ public class GeoDistanceSortParser implements SortParser {
         } else {
             objectMapper = context.mapperService().resolveClosestNestedObjectMapper(fieldName);
         }
+        final Nested nested;
         if (objectMapper != null && objectMapper.nested().isNested()) {
             Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE);
             Filter innerDocumentsFilter;
@@ -169,11 +149,45 @@ public class GeoDistanceSortParser implements SortParser {
             } else {
                 innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter());
             }
-            geoDistanceComparatorSource = new NestedFieldComparatorSource(
+            nested = new Nested(rootDocumentsFilter, innerDocumentsFilter);
-                sortMode, geoDistanceComparatorSource, rootDocumentsFilter, innerDocumentsFilter
+        } else {
-            );
+            nested = null;
         }
 
+        IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
+
+            @Override
+            public SortField.Type reducedType() {
+                return SortField.Type.DOUBLE;
+            }
+
+            @Override
+            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+                return new FieldComparator.DoubleComparator(numHits, null, null, null) {
+                    @Override
+                    protected Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException {
+                        final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
+                        final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(distance, geoPointValues);
+                        final NumericDoubleValues selectedValues;
+                        if (nested == null) {
+                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE);
+                        } else {
+                            final FixedBitSet rootDocs = nested.rootDocs(context);
+                            final FixedBitSet innerDocs = nested.innerDocs(context);
+                            selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, context.reader().maxDoc());
+                        }
+                        return new Doubles() {
+                            @Override
+                            public double get(int docID) {
+                                return selectedValues.get(docID);
+                            }
+                        };
+                    }
+                };
+            }
+
+        };
+
         return new SortField(fieldName, geoDistanceComparatorSource, reverse);
     }
 }

src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java

@@ -19,21 +19,25 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleScriptDataComparator;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.fieldcomparator.StringScriptDataComparator;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
 import org.elasticsearch.index.mapper.ObjectMappers;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.index.query.ParsedFilter;
-import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
 import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -44,6 +48,9 @@ import java.util.Map;
  */
 public class ScriptSortParser implements SortParser {
 
+    private static final String STRING_SORT_TYPE = "string";
+    private static final String NUMBER_SORT_TYPE = "number";
+
     @Override
     public String[] names() {
         return new String[]{"_script"};
@@ -105,17 +112,9 @@ public class ScriptSortParser implements SortParser {
         if (type == null) {
             throw new SearchParseException(context, "_script sorting requires setting the type of the script");
         }
-        SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params);
+        final SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params);
-        IndexFieldData.XFieldComparatorSource fieldComparatorSource;
-        if ("string".equals(type)) {
-            fieldComparatorSource = StringScriptDataComparator.comparatorSource(searchScript);
-        } else if ("number".equals(type)) {
-            fieldComparatorSource = DoubleScriptDataComparator.comparatorSource(searchScript);
-        } else {
-            throw new SearchParseException(context, "custom script sort type [" + type + "] not supported");
-        }
 
-        if ("string".equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) {
+        if (STRING_SORT_TYPE.equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) {
             throw new SearchParseException(context, "type [string] doesn't support mode [" + sortMode + "]");
         }
 
@@ -125,6 +124,7 @@ public class ScriptSortParser implements SortParser {
 
         // If nested_path is specified, then wrap the `fieldComparatorSource` in a `NestedFieldComparatorSource`
         ObjectMapper objectMapper;
+        final Nested nested;
         if (nestedPath != null) {
             ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath);
             if (objectMappers == null) {
@@ -142,7 +142,58 @@ public class ScriptSortParser implements SortParser {
             } else {
                 innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter());
             }
-            fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter);
+            nested = new Nested(rootDocumentsFilter, innerDocumentsFilter);
+        } else {
+            nested = null;
+        }
+
+        final IndexFieldData.XFieldComparatorSource fieldComparatorSource;
+        switch (type) {
+            case STRING_SORT_TYPE:
+                fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, sortMode, nested) {
+                    @Override
+                    protected SortedBinaryDocValues getValues(AtomicReaderContext context) {
+                        searchScript.setNextReader(context);
+                        final BinaryDocValues values = new BinaryDocValues() {
+                            final BytesRef spare = new BytesRef();
+                            @Override
+                            public BytesRef get(int docID) {
+                                searchScript.setNextDocId(docID);
+                                spare.copyChars(searchScript.run().toString());
+                                return spare;
+                            }
+                        };
+                        return FieldData.singleton(values, null);
+                    }
+                    @Override
+                    protected void setScorer(Scorer scorer) {
+                        searchScript.setScorer(scorer);
+                    }
+                };
+                break;
+            case NUMBER_SORT_TYPE:
+                // TODO: should we rather sort missing values last?
+                fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, sortMode, nested) {
+                    @Override
+                    protected SortedNumericDoubleValues getValues(AtomicReaderContext context) {
+                        searchScript.setNextReader(context);
+                        final NumericDoubleValues values = new NumericDoubleValues() {
+                            @Override
+                            public double get(int docID) {
+                                searchScript.setNextDocId(docID);
+                                return searchScript.runAsDouble();
+                            }
+                        };
+                        return FieldData.singleton(values, null);
+                    }
+                    @Override
+                    protected void setScorer(Scorer scorer) {
+                        searchScript.setScorer(scorer);
+                    }
+                };
+                break;
+            default:
+                throw new SearchParseException(context, "custom script sort type [" + type + "] not supported");
         }
 
         return new SortField("_script", fieldComparatorSource, reverse);

src/main/java/org/elasticsearch/search/sort/SortParseElement.java

@@ -28,14 +28,14 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.ObjectMappers;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.index.query.ParsedFilter;
-import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
 import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.internal.SearchContext;
@@ -190,7 +190,7 @@ public class SortParseElement implements SearchParseElement {
                 sortFields.add(SORT_DOC);
             }
         } else {
-            FieldMapper fieldMapper = context.smartNameFieldMapper(fieldName);
+            FieldMapper<?> fieldMapper = context.smartNameFieldMapper(fieldName);
             if (fieldMapper == null) {
                 if (ignoreUnmapped) {
                     return;
@@ -219,8 +219,7 @@ public class SortParseElement implements SearchParseElement {
                 sortMode = resolveDefaultSortMode(reverse);
             }
 
-            IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper)
+
-                    .comparatorSource(missing, sortMode);
             ObjectMapper objectMapper;
             if (nestedPath != null) {
                 ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath);
@@ -234,6 +233,7 @@ public class SortParseElement implements SearchParseElement {
             } else {
                 objectMapper = context.mapperService().resolveClosestNestedObjectMapper(fieldName);
             }
+            final Nested nested;
             if (objectMapper != null && objectMapper.nested().isNested()) {
                 Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE);
                 Filter innerDocumentsFilter;
@@ -242,8 +242,13 @@ public class SortParseElement implements SearchParseElement {
                 } else {
                     innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter());
                 }
-                fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter);
+                nested = new Nested(rootDocumentsFilter, innerDocumentsFilter);
+            } else {
+                nested = null;
             }
+
+            IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper)
+                    .comparatorSource(missing, sortMode, nested);
             sortFields.add(new SortField(fieldMapper.names().indexName(), fieldComparatorSource, reverse));
         }
     }

src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java

@@ -28,17 +28,18 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.test.ElasticsearchTestCase;
+import org.elasticsearch.test.ElasticsearchTestCase.UsesLuceneFieldCacheOnPurpose;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.ArrayList;
 
-import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
  *
  */
+@UsesLuceneFieldCacheOnPurpose
 public class SimpleLuceneTests extends ElasticsearchTestCase {
 
     @Test

src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java

@@ -24,7 +24,7 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.search.*;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
 import org.elasticsearch.search.MultiValueMode;
 import org.junit.Test;
 
@@ -100,7 +100,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         TopFieldDocs topDocs;
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
@@ -110,7 +110,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@@ -167,14 +167,14 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertValues(bytesValues, 2, three());
         
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
-        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
+        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs.length, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
 
-        topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs.length, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
@@ -226,7 +226,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
 
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
         TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@@ -242,12 +242,12 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertThat(topDocs.scoreDocs[5].doc, equalTo(6));
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("08"));
         assertThat(topDocs.scoreDocs[6].doc, equalTo(1));
-        assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM));
+        assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(XFieldComparatorSource.MAX_TERM));
         assertThat(topDocs.scoreDocs[7].doc, equalTo(5));
-        assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM));
+        assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(XFieldComparatorSource.MAX_TERM));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(6));

src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTests.java

@@ -91,14 +91,14 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
         TopFieldDocs topDocs;
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@@ -145,42 +145,42 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
         TopFieldDocs topDocs;
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN))));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX), true)));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(0));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MIN))));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MAX), true)));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@@ -323,7 +323,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
 
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
         TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@@ -344,7 +344,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
 //        assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
@@ -366,7 +366,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
 
         searcher = new IndexSearcher(DirectoryReader.open(writer, true));
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM)))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null)))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@@ -388,7 +388,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
 
         searcher = new IndexSearcher(DirectoryReader.open(writer, true));
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM), true))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null), true))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
@@ -410,7 +410,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
 
         searcher = new IndexSearcher(DirectoryReader.open(writer, true));
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG)))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null)))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@@ -432,7 +432,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
 
         searcher = new IndexSearcher(DirectoryReader.open(writer, true));
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG), true))); // defaults to _last
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null), true))); // defaults to _last
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
@@ -453,7 +453,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
 //        assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN))));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
@@ -466,7 +466,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
         assertThat(topDocs.scoreDocs[7].doc, equalTo(6));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX), true)));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
@@ -479,7 +479,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
         assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("-9", MultiValueMode.MIN))));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("-9", MultiValueMode.MIN, null))));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@@ -492,7 +492,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
         assertThat(topDocs.scoreDocs[7].doc, equalTo(6));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
-                new Sort(new SortField("value", indexFieldData.comparatorSource("9", MultiValueMode.MAX), true)));
+                new Sort(new SortField("value", indexFieldData.comparatorSource("9", MultiValueMode.MAX, null), true)));
         assertThat(topDocs.totalHits, equalTo(8));
         assertThat(topDocs.scoreDocs.length, equalTo(8));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(6));

src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTests.java

@@ -35,9 +35,9 @@ import org.elasticsearch.common.lucene.search.NotFilter;
 import org.elasticsearch.common.lucene.search.XFilteredQuery;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
-import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
 import org.elasticsearch.search.MultiValueMode;
 import org.junit.Test;
 
@@ -237,7 +237,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
         final IndexFieldData indexFieldData = getForField("value");
         final String missingValue = values[1];
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
-        XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, MultiValueMode.MIN);
+        XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, MultiValueMode.MIN, null);
         TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
         assertEquals(numDocs, topDocs.totalHits);
         BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
@@ -293,7 +293,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
         }
         final IndexFieldData indexFieldData = getForField("value");
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
-        XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", MultiValueMode.MIN);
+        XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", MultiValueMode.MIN, null);
         TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
         assertEquals(numDocs, topDocs.totalHits);
         BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
@@ -360,13 +360,13 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
         }
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
         IndexFieldData<?> fieldData = getForField("text");
-        final BytesRef missingValue;
+        final Object missingValue;
         switch (randomInt(4)) {
         case 0:
-            missingValue = new BytesRef();
+            missingValue = "_first";
             break;
         case 1:
-            missingValue = BytesRefFieldComparatorSource.MAX_TERM;
+            missingValue = "_last";
             break;
         case 2:
             missingValue = new BytesRef(RandomPicks.randomFrom(getRandom(), values));
@@ -375,10 +375,10 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
             missingValue = new BytesRef(TestUtil.randomSimpleString(getRandom()));
             break;
         }
-        BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode);
         Filter parentFilter = new TermFilter(new Term("type", "parent"));
         Filter childFilter = new NotFilter(parentFilter);
-        NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
+        Nested nested = new Nested(parentFilter, childFilter);
+        BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested);
         ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
         Sort sort = new Sort(new SortField("text", nestedComparatorSource));
         TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort);
@@ -389,26 +389,36 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
             assertTrue("expected " + docID + " to be a parent", parents.get(docID));
             BytesRef cmpValue = null;
             for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) {
-                String[] vals = searcher.doc(child).getValues("text");
+                String[] sVals = searcher.doc(child).getValues("text");
-                if (vals.length == 0) {
+                final BytesRef[] vals;
-                    vals = new String[] {missingValue.utf8ToString()};
+                if (sVals.length == 0) {
+                    vals = new BytesRef[0];
+                } else {
+                    vals = new BytesRef[sVals.length];
+                    for (int j = 0; j < vals.length; ++j) {
+                        vals[j] = new BytesRef(sVals[j]);
+                    }
                 }
-                for (String value : vals) {
+                for (BytesRef value : vals) {
-                    final BytesRef bytesValue = new BytesRef(value);
                     if (cmpValue == null) {
-                        cmpValue = bytesValue;
+                        cmpValue = value;
-                    } else if (sortMode == MultiValueMode.MIN && bytesValue.compareTo(cmpValue) < 0) {
+                    } else if (sortMode == MultiValueMode.MIN && value.compareTo(cmpValue) < 0) {
-                        cmpValue = bytesValue;
+                        cmpValue = value;
-                    } else if (sortMode == MultiValueMode.MAX && bytesValue.compareTo(cmpValue) > 0) {
+                    } else if (sortMode == MultiValueMode.MAX && value.compareTo(cmpValue) > 0) {
-                        cmpValue = bytesValue;
+                        cmpValue = value;
                     }
                 }
             }
             if (cmpValue == null) {
-                cmpValue = missingValue;
+                if ("_first".equals(missingValue)) {
+                    cmpValue = new BytesRef();
+                } else if ("_last".equals(missingValue)) {
+                    cmpValue = XFieldComparatorSource.MAX_TERM;
+                } else {
+                    cmpValue = (BytesRef) missingValue;
+                }
             }
             if (previous != null) {
-                assertNotNull(cmpValue);
                 assertTrue(previous.utf8ToString() + "   /   " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0);
             }
             previous = cmpValue;

src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java

@@ -22,6 +22,7 @@ package org.elasticsearch.index.fielddata;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
 import org.elasticsearch.search.MultiValueMode;
@@ -60,8 +61,8 @@ public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTes
             }
 
             @Override
-            public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
+            public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
-                return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
+                return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
             }
 
             @Override

src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java

@@ -27,6 +27,7 @@ import org.apache.lucene.search.*;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
@@ -140,7 +141,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTests {
     public void testSorting() throws Exception {
         IndexFieldData indexFieldData = getForField(childType);
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
-        IndexFieldData.XFieldComparatorSource comparator = indexFieldData.comparatorSource("_last", MultiValueMode.MIN);
+        IndexFieldData.XFieldComparatorSource comparator = indexFieldData.comparatorSource("_last", MultiValueMode.MIN, null);
 
         TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.NAME, comparator, false)));
         assertThat(topDocs.totalHits, equalTo(8));
@@ -160,7 +161,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTests {
         assertThat(topDocs.scoreDocs[6].doc, equalTo(6));
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("2"));
         assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
-        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(IndexFieldData.XFieldComparatorSource.MAX_TERM));
+        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(XFieldComparatorSource.MAX_TERM));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.NAME, comparator, true)));
         assertThat(topDocs.totalHits, equalTo(8));

src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java

@@ -108,7 +108,7 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
         });
         Ordinals ords = creationMultiOrdinals(builder);
         RandomAccessOrds docs = ords.ordinals();
-        final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs, -1);
+        final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs);
         int docId = ordsAndIds.get(0).id;
         List<Long> docOrds = new ArrayList<>();
         for (OrdAndId ordAndId : ordsAndIds) {

src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTests.java

@@ -33,6 +33,8 @@ import org.elasticsearch.common.lucene.search.NotFilter;
 import org.elasticsearch.common.lucene.search.XFilteredQuery;
 import org.elasticsearch.index.fielddata.AbstractFieldDataTests;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.search.MultiValueMode;
 import org.junit.Test;
 
@@ -206,10 +208,9 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
 
         MultiValueMode sortMode = MultiValueMode.SUM;
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false));
-        IndexFieldData.XFieldComparatorSource innerFieldComparator = createInnerFieldComparator("field2", sortMode, null);
         Filter parentFilter = new TermFilter(new Term("__type", "parent"));
         Filter childFilter = new NotFilter(parentFilter);
-        NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
+        XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, null, new Nested(parentFilter, childFilter));
         ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
 
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
@@ -243,7 +244,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9));
 
         childFilter = new TermFilter(new Term("filter_1", "T"));
-        nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
+        nestedComparatorSource = createFieldComparator("field2", sortMode, null, new Nested(parentFilter, childFilter));
         query = new ToParentBlockJoinQuery(
                 new XFilteredQuery(new MatchAllDocsQuery(), childFilter),
                 new FixedBitSetCachingWrapperFilter(parentFilter),
@@ -279,8 +280,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
         assertThat(topDocs.scoreDocs[4].doc, equalTo(3));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9));
 
-        innerFieldComparator = createInnerFieldComparator("field2", sortMode, 127);
+        nestedComparatorSource = createFieldComparator("field2", sortMode, 127, new Nested(parentFilter, childFilter));
-        nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
         sort = new Sort(new SortField("field2", nestedComparatorSource, true));
         topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort);
         assertThat(topDocs.totalHits, equalTo(8));
@@ -296,8 +296,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
         assertThat(topDocs.scoreDocs[4].doc, equalTo(7));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(8));
 
-        innerFieldComparator = createInnerFieldComparator("field2", sortMode, -127);
+        nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
-        nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
         sort = new Sort(new SortField("field2", nestedComparatorSource));
         topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort);
         assertThat(topDocs.totalHits, equalTo(8));
@@ -315,33 +314,33 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
 
         // Moved to method, because floating point based XFieldComparatorSource have different outcome for SortMode avg,
         // than integral number based implementations...
-        assertAvgScoreMode(parentFilter, searcher, innerFieldComparator);
+        assertAvgScoreMode(parentFilter, searcher);
         searcher.getIndexReader().close();
     }
 
-    protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException {
+    protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException {
         MultiValueMode sortMode = MultiValueMode.AVG;
         Filter childFilter = new NotFilter(parentFilter);
-        NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
+        XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
         Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopDocs topDocs = searcher.search(query, 5, sort);
         assertThat(topDocs.totalHits, equalTo(7));
         assertThat(topDocs.scoreDocs.length, equalTo(5));
-        assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(11));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(2));
-        assertThat(topDocs.scoreDocs[1].doc, equalTo(11));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(3));
-        assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(2));
+        assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(3));
-        assertThat(topDocs.scoreDocs[2].doc, equalTo(3));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(7));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(3));
         assertThat(topDocs.scoreDocs[3].doc, equalTo(15));
         assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(3));
         assertThat(topDocs.scoreDocs[4].doc, equalTo(19));
-        assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(3));
+        assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(4));
     }
 
     protected abstract IndexableField createField(String name, int value, Field.Store store);
 
-    protected abstract IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue);
+    protected abstract IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested);
 
 }

src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java

@@ -29,9 +29,11 @@ import org.elasticsearch.common.lucene.search.NotFilter;
 import org.elasticsearch.common.lucene.search.XFilteredQuery;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
@@ -47,9 +49,9 @@ public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTests {
     }
 
     @Override
-    protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) {
+    protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) {
         DoubleArrayIndexFieldData fieldData = getForField(fieldName);
-        return new DoubleValuesComparatorSource(fieldData, missingValue, sortMode);
+        return new DoubleValuesComparatorSource(fieldData, missingValue, sortMode, nested);
     }
 
     @Override
@@ -57,10 +59,10 @@ public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTests {
         return new DoubleField(name, value, store);
     }
 
-    protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException {
+    protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException {
         MultiValueMode sortMode = MultiValueMode.AVG;
         Filter childFilter = new NotFilter(parentFilter);
-        NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
+        XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
         Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopDocs topDocs = searcher.search(query, 5, sort);

src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java

@@ -29,9 +29,11 @@ import org.elasticsearch.common.lucene.search.NotFilter;
 import org.elasticsearch.common.lucene.search.XFilteredQuery;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.index.fielddata.plain.FloatArrayIndexFieldData;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
@@ -39,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo;
 
 /**
  */
-public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests {
+public class FloatNestedSortingTests extends DoubleNestedSortingTests {
 
     @Override
     protected FieldDataType getFieldDataType() {
@@ -47,9 +49,9 @@ public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests {
     }
 
     @Override
-    protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) {
+    protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) {
         FloatArrayIndexFieldData fieldData = getForField(fieldName);
-        return new FloatValuesComparatorSource(fieldData, missingValue, sortMode);
+        return new FloatValuesComparatorSource(fieldData, missingValue, sortMode, nested);
     }
 
     @Override
@@ -60,7 +62,7 @@ public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests {
     protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException {
         MultiValueMode sortMode = MultiValueMode.AVG;
         Filter childFilter = new NotFilter(parentFilter);
-        NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
+        XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
         Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
         TopDocs topDocs = searcher.search(query, 5, sort);

src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java

@@ -23,9 +23,10 @@ import org.apache.lucene.document.LongField;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.index.fielddata.plain.PackedArrayIndexFieldData;
+import org.elasticsearch.search.MultiValueMode;
 
 /**
  */
@@ -37,9 +38,9 @@ public class LongNestedSortingTests extends AbstractNumberNestedSortingTests {
     }
 
     @Override
-    protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) {
+    protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) {
         PackedArrayIndexFieldData fieldData = getForField(fieldName);
-        return new LongValuesComparatorSource(fieldData, missingValue, sortMode);
+        return new LongValuesComparatorSource(fieldData, missingValue, sortMode, nested);
     }
 
     @Override

src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java

@@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.search.XFilteredQuery;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.search.MultiValueMode;
@@ -106,8 +107,7 @@ public class NestedSortingTests extends AbstractFieldDataTests {
     private TopDocs getTopDocs(IndexSearcher searcher, IndexFieldData<?> indexFieldData, String missingValue, MultiValueMode sortMode, int n, boolean reverse) throws IOException {
         Filter parentFilter = new TermFilter(new Term("__type", "parent"));
         Filter childFilter = new TermFilter(new Term("__type", "child"));
-        XFieldComparatorSource innerSource = indexFieldData.comparatorSource(missingValue, sortMode);
+        XFieldComparatorSource nestedComparatorSource = indexFieldData.comparatorSource(missingValue, sortMode, new Nested(parentFilter, childFilter));
-        NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
         Query query = new ConstantScoreQuery(parentFilter);
         Sort sort = new Sort(new SortField("f", nestedComparatorSource, reverse));
         return searcher.search(query, n, sort);
@@ -271,10 +271,9 @@ public class NestedSortingTests extends AbstractFieldDataTests {
         MultiValueMode sortMode = MultiValueMode.MIN;
         IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false));
         PagedBytesIndexFieldData indexFieldData = getForField("field2");
-        BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode);
         Filter parentFilter = new TermFilter(new Term("__type", "parent"));
         Filter childFilter = new NotFilter(parentFilter);
-        NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
+        BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter));
         ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
 
         Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
@@ -293,7 +292,7 @@ public class NestedSortingTests extends AbstractFieldDataTests {
         assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("i"));
 
         sortMode = MultiValueMode.MAX;
-        nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
+        nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter));
         sort = new Sort(new SortField("field2", nestedComparatorSource, true));
         topDocs = searcher.search(query, 5, sort);
         assertThat(topDocs.totalHits, equalTo(7));
@@ -311,7 +310,7 @@ public class NestedSortingTests extends AbstractFieldDataTests {
 
 
         childFilter = new AndFilter(Arrays.asList(new NotFilter(parentFilter), new TermFilter(new Term("filter_1", "T"))));
-        nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
+        nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter));
         query = new ToParentBlockJoinQuery(
                 new XFilteredQuery(new MatchAllDocsQuery(), childFilter),
                 new FixedBitSetCachingWrapperFilter(parentFilter),

src/test/java/org/elasticsearch/nested/SimpleNestedTests.java

@@ -1263,11 +1263,11 @@ public class SimpleNestedTests extends ElasticsearchIntegrationTest {
         assertHitCount(searchResponse, 3);
         assertThat(searchResponse.getHits().getHits().length, equalTo(3));
         assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("3"));
-        assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("0"));
+        assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1"));
         assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
-        assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("1"));
+        assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2"));
         assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("1"));
-        assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("2"));
+        assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3"));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
@@ -1282,11 +1282,11 @@ public class SimpleNestedTests extends ElasticsearchIntegrationTest {
         assertHitCount(searchResponse, 3);
         assertThat(searchResponse.getHits().getHits().length, equalTo(3));
         assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
-        assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("2"));
+        assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("3"));
         assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
-        assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("1"));
+        assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2"));
         assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
-        assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("0"));
+        assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("1"));
 
         // Sort mode: avg with filter
         searchResponse = client().prepareSearch()

src/test/java/org/elasticsearch/search/MultiValueModeTests.java

@@ -0,0 +1,545 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.apache.lucene.index.*;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.test.ElasticsearchTestCase;
+
+import java.util.Arrays;
+
+public class MultiValueModeTests extends ElasticsearchTestCase {
+
+    private static FixedBitSet randomRootDocs(int maxDoc) {
+        FixedBitSet set = new FixedBitSet(maxDoc);
+        for (int i = 0; i < maxDoc; ++i) {
+            if (randomBoolean()) {
+                set.set(i);
+            }
+        }
+        // the last doc must be a root doc
+        set.set(maxDoc - 1);
+        return set;
+    }
+
+    private static FixedBitSet randomInnerDocs(FixedBitSet rootDocs) {
+        FixedBitSet innerDocs = new FixedBitSet(rootDocs.length());
+        for (int i = 0; i < innerDocs.length(); ++i) {
+            if (!rootDocs.get(i) && randomBoolean()) {
+                innerDocs.set(i);
+            }
+        }
+        return innerDocs;
+    }
+
+    public void testSingleValuedLongs() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final long[] array = new long[numDocs];
+        final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
+        for (int i = 0; i < array.length; ++i) {
+            if (randomBoolean()) {
+                array[i] = randomLong();
+                if (docsWithValue != null) {
+                    docsWithValue.set(i);
+                }
+            } else if (docsWithValue != null && randomBoolean()) {
+                docsWithValue.set(i);
+            }
+        }
+        final NumericDocValues singleValues = new NumericDocValues() {
+            @Override
+            public long get(int docID) {
+                return array[docID];
+            }
+        };
+        final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue);
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    public void testMultiValuedLongs() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final long[][] array = new long[numDocs][];
+        for (int i = 0; i < numDocs; ++i) {
+            final long[] values = new long[randomInt(4)];
+            for (int j = 0; j < values.length; ++j) {
+                values[j] = randomLong();
+            }
+            Arrays.sort(values);
+            array[i] = values;
+        }
+        final SortedNumericDocValues multiValues = new SortedNumericDocValues() {
+            int doc;
+
+            @Override
+            public long valueAt(int index) {
+                return array[doc][index];
+            }
+
+            @Override
+            public void setDocument(int doc) {
+                this.doc = doc;
+            }
+
+            @Override
+            public int count() {
+                return array[doc].length;
+            }
+        };
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    private void verify(SortedNumericDocValues values, int maxDoc) {
+        for (long missingValue : new long[] { 0, randomLong() }) {
+            for (MultiValueMode mode : MultiValueMode.values()) {
+                final NumericDocValues selected = mode.select(values, missingValue);
+                for (int i = 0; i < maxDoc; ++i) {
+                    final long actual = selected.get(i);
+                    long expected;
+                    values.setDocument(i);
+                    int numValues = values.count();
+                    if (numValues == 0) {
+                        expected = missingValue;
+                    } else {
+                        expected = mode.startLong();
+                        for (int j = 0; j < numValues; ++j) {
+                            expected = mode.apply(expected, values.valueAt(j));
+                        }
+                        expected = mode.reduce(expected, numValues);
+                    }
+
+                    assertEquals(mode.toString() + " docId=" + i, expected, actual);
+                }
+            }
+        }
+    }
+
+    private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
+        for (long missingValue : new long[] { 0, randomLong() }) {
+            for (MultiValueMode mode : MultiValueMode.values()) {
+                final NumericDocValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc);
+                int prevRoot = -1;
+                for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
+                    final long actual = selected.get(root);
+                    long expected = mode.startLong();
+                    int numValues = 0;
+                    for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
+                        values.setDocument(child);
+                        for (int j = 0; j < values.count(); ++j) {
+                            expected = mode.apply(expected, values.valueAt(j));
+                            ++numValues;
+                        }
+                    }
+                    if (numValues == 0) {
+                        expected = missingValue;
+                    } else {
+                        expected = mode.reduce(expected, numValues);
+                    }
+
+                    assertEquals(mode.toString() + " docId=" + root, expected, actual);
+
+                    prevRoot = root;
+                }
+            }
+        }
+    }
+
+    public void testSingleValuedDoubles() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final double[] array = new double[numDocs];
+        final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
+        for (int i = 0; i < array.length; ++i) {
+            if (randomBoolean()) {
+                array[i] = randomDouble();
+                if (docsWithValue != null) {
+                    docsWithValue.set(i);
+                }
+            } else if (docsWithValue != null && randomBoolean()) {
+                docsWithValue.set(i);
+            }
+        }
+        final NumericDoubleValues singleValues = new NumericDoubleValues() {
+            @Override
+            public double get(int docID) {
+                return array[docID];
+            }
+        };
+        final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues, docsWithValue);
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    public void testMultiValuedDoubles() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final double[][] array = new double[numDocs][];
+        for (int i = 0; i < numDocs; ++i) {
+            final double[] values = new double[randomInt(4)];
+            for (int j = 0; j < values.length; ++j) {
+                values[j] = randomDouble();
+            }
+            Arrays.sort(values);
+            array[i] = values;
+        }
+        final SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() {
+            int doc;
+
+            @Override
+            public double valueAt(int index) {
+                return array[doc][index];
+            }
+
+            @Override
+            public void setDocument(int doc) {
+                this.doc = doc;
+            }
+
+            @Override
+            public int count() {
+                return array[doc].length;
+            }
+        };
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    private void verify(SortedNumericDoubleValues values, int maxDoc) {
+        for (long missingValue : new long[] { 0, randomLong() }) {
+            for (MultiValueMode mode : MultiValueMode.values()) {
+                final NumericDoubleValues selected = mode.select(values, missingValue);
+                for (int i = 0; i < maxDoc; ++i) {
+                    final double actual = selected.get(i);
+                    double expected;
+                    values.setDocument(i);
+                    int numValues = values.count();
+                    if (numValues == 0) {
+                        expected = missingValue;
+                    } else {
+                        expected = mode.startLong();
+                        for (int j = 0; j < numValues; ++j) {
+                            expected = mode.apply(expected, values.valueAt(j));
+                        }
+                        expected = mode.reduce(expected, numValues);
+                    }
+
+                    assertEquals(mode.toString() + " docId=" + i, expected, actual, 0.1);
+                }
+            }
+        }
+    }
+
+    private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
+        for (long missingValue : new long[] { 0, randomLong() }) {
+            for (MultiValueMode mode : MultiValueMode.values()) {
+                final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc);
+                int prevRoot = -1;
+                for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
+                    final double actual = selected.get(root);
+                    double expected = mode.startLong();
+                    int numValues = 0;
+                    for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
+                        values.setDocument(child);
+                        for (int j = 0; j < values.count(); ++j) {
+                            expected = mode.apply(expected, values.valueAt(j));
+                            ++numValues;
+                        }
+                    }
+                    if (numValues == 0) {
+                        expected = missingValue;
+                    } else {
+                        expected = mode.reduce(expected, numValues);
+                    }
+
+                    assertEquals(mode.toString() + " docId=" + root, expected, actual, 0.1);
+
+                    prevRoot = root;
+                }
+            }
+        }
+    }
+
+    public void testSingleValuedStrings() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final BytesRef[] array = new BytesRef[numDocs];
+        final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
+        for (int i = 0; i < array.length; ++i) {
+            if (randomBoolean()) {
+                array[i] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8));
+                if (docsWithValue != null) {
+                    docsWithValue.set(i);
+                }
+            } else {
+                array[i] = new BytesRef();
+                if (docsWithValue != null && randomBoolean()) {
+                    docsWithValue.set(i);
+                }
+            }
+        }
+        final BinaryDocValues singleValues = new BinaryDocValues() {
+            @Override
+            public BytesRef get(int docID) {
+                return BytesRef.deepCopyOf(array[docID]);
+            }
+        };
+        final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue);
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    public void testMultiValuedStrings() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final BytesRef[][] array = new BytesRef[numDocs][];
+        for (int i = 0; i < numDocs; ++i) {
+            final BytesRef[] values = new BytesRef[randomInt(4)];
+            for (int j = 0; j < values.length; ++j) {
+                values[j] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8));
+            }
+            Arrays.sort(values);
+            array[i] = values;
+        }
+        final SortedBinaryDocValues multiValues = new SortedBinaryDocValues() {
+            int doc;
+
+            @Override
+            public BytesRef valueAt(int index) {
+                return BytesRef.deepCopyOf(array[doc][index]);
+            }
+
+            @Override
+            public void setDocument(int doc) {
+                this.doc = doc;
+            }
+
+            @Override
+            public int count() {
+                return array[doc].length;
+            }
+        };
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    private void verify(SortedBinaryDocValues values, int maxDoc) {
+        for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) {
+            for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
+                final BinaryDocValues selected = mode.select(values, missingValue);
+                for (int i = 0; i < maxDoc; ++i) {
+                    final BytesRef actual = selected.get(i);
+                    BytesRef expected = null;
+                    values.setDocument(i);
+                    int numValues = values.count();
+                    if (numValues == 0) {
+                        expected = missingValue;
+                    } else {
+                        for (int j = 0; j < numValues; ++j) {
+                            if (expected == null) {
+                                expected = BytesRef.deepCopyOf(values.valueAt(j));
+                            } else {
+                                expected = mode.apply(expected, BytesRef.deepCopyOf(values.valueAt(j)));
+                            }
+                        }
+                        if (expected == null) {
+                            expected = missingValue;
+                        }
+                    }
+
+                    assertEquals(mode.toString() + " docId=" + i, expected, actual);
+                }
+            }
+        }
+    }
+
+    private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
+        for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) {
+            for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
+                final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc);
+                int prevRoot = -1;
+                for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
+                    final BytesRef actual = selected.get(root);
+                    BytesRef expected = null;
+                    for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
+                        values.setDocument(child);
+                        for (int j = 0; j < values.count(); ++j) {
+                            if (expected == null) {
+                                expected = BytesRef.deepCopyOf(values.valueAt(j));
+                            } else {
+                                expected = mode.apply(expected, values.valueAt(j));
+                            }
+                            expected = mode.apply(expected, BytesRef.deepCopyOf(values.valueAt(j)));
+                        }
+                    }
+                    if (expected == null) {
+                        expected = missingValue;
+                    }
+
+                    assertEquals(mode.toString() + " docId=" + root, expected, actual);
+
+                    prevRoot = root;
+                }
+            }
+        }
+    }
+
+
+    public void testSingleValuedOrds() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final int[] array = new int[numDocs];
+        for (int i = 0; i < array.length; ++i) {
+            if (randomBoolean()) {
+                array[i] = randomInt(1000);
+            } else {
+                array[i] = -1;
+            }
+        }
+        final SortedDocValues singleValues = new SortedDocValues() {
+            @Override
+            public int getOrd(int docID) {
+                return array[docID];
+            }
+
+            @Override
+            public BytesRef lookupOrd(int ord) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public int getValueCount() {
+                return 1 << 20;
+            }
+        };
+        final RandomAccessOrds multiValues = (RandomAccessOrds) DocValues.singleton(singleValues);
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    public void testMultiValuedOrds() {
+        final int numDocs = scaledRandomIntBetween(1, 100);
+        final long[][] array = new long[numDocs][];
+        for (int i = 0; i < numDocs; ++i) {
+            final long[] values = new long[randomInt(4)];
+            for (int j = 0; j < values.length; ++j) {
+                values[j] = j == 0 ? randomInt(1000) : values[j - 1] + 1 + randomInt(1000);
+            }
+            array[i] = values;
+        }
+        final RandomAccessOrds multiValues = new RandomAccessOrds() {
+            int doc;
+
+            @Override
+            public long ordAt(int index) {
+                return array[doc][index];
+            }
+
+            @Override
+            public int cardinality() {
+                return array[doc].length;
+            }
+
+            @Override
+            public long nextOrd() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public void setDocument(int docID) {
+                this.doc = docID;
+            }
+
+            @Override
+            public BytesRef lookupOrd(long ord) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public long getValueCount() {
+                return 1 << 20;
+            }
+        };
+        verify(multiValues, numDocs);
+        final FixedBitSet rootDocs = randomRootDocs(numDocs);
+        final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
+        verify(multiValues, numDocs, rootDocs, innerDocs);
+    }
+
+    private void verify(RandomAccessOrds values, int maxDoc) {
+        for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
+            final SortedDocValues selected = mode.select(values);
+            for (int i = 0; i < maxDoc; ++i) {
+                final long actual = selected.getOrd(i);
+                int expected = -1;
+                values.setDocument(i);
+                for (int j = 0; j < values.cardinality(); ++j) {
+                    if (expected == -1) {
+                        expected = (int) values.ordAt(j);
+                    } else {
+                        expected = mode.applyOrd(expected, (int) values.ordAt(j));
+                    }
+                }
+
+                assertEquals(mode.toString() + " docId=" + i, expected, actual);
+            }
+        }
+    }
+
+    private void verify(RandomAccessOrds values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
+        for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
+            final SortedDocValues selected = mode.select(values, rootDocs, innerDocs);
+            int prevRoot = -1;
+            for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
+                final int actual = selected.getOrd(root);
+                int expected = -1;
+                for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
+                    values.setDocument(child);
+                    for (int j = 0; j < values.cardinality(); ++j) {
+                        if (expected == -1) {
+                            expected = (int) values.ordAt(j);
+                        } else {
+                            expected = mode.applyOrd(expected, (int) values.ordAt(j));
+                        }
+                    }
+                }
+
+                assertEquals(mode.toString() + " docId=" + root, expected, actual);
+
+                prevRoot = root;
+            }
+        }
+    }
+}

src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java

@@ -23,6 +23,7 @@ import com.carrotsearch.randomizedtesting.annotations.*;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
 import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableList;
+import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.AbstractRandomizedTest;
 import org.apache.lucene.util.LuceneTestCase;
@@ -39,10 +40,7 @@ import org.elasticsearch.test.cache.recycler.MockBigArrays;
 import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.store.MockDirectoryHelper;
-import org.junit.After;
+import org.junit.*;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
 
 import java.io.Closeable;
 import java.io.File;
@@ -99,6 +97,21 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
 
     }
 
+    @Before
+    public void cleanFieldCache() {
+        FieldCache.DEFAULT.purgeAllCaches();
+    }
+
+    @After
+    public void ensureNoFieldCacheUse() {
+        // We use the lucene comparators, and by default they work on field cache.
+        // However, given the way that we use them, field cache should NEVER get loaded.
+        if (getClass().getAnnotation(UsesLuceneFieldCacheOnPurpose.class) == null) {
+            FieldCache.CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
+            assertEquals("fieldcache must never be used, got=" + Arrays.toString(entries), 0, entries.length);
+        }
+    }
+
     /**
      * Runs the code block for 10 seconds waiting for no assertion to trip.
      */
@@ -395,6 +408,15 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
         int version();
     }
 
+    /**
+     * Most tests don't use {@link FieldCache} but some of them might do.
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target({ElementType.TYPE})
+    @Ignore
+    public @interface UsesLuceneFieldCacheOnPurpose {
+    }
+
     /**
      * Returns a global compatibility version that is set via the
      * {@value #TESTS_COMPATIBILITY} or {@value #TESTS_BACKWARDS_COMPATIBILITY_VERSION} system property.