initial commit

src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.lucene;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A wrapped to {@link BytesRef} that also caches the hashCode for it.
+ */
+public class HashedBytesRef {
+
+    public BytesRef bytes;
+    public int hash;
+
+    public HashedBytesRef() {
+    }
+
+    public HashedBytesRef(String bytes) {
+        this(new BytesRef(bytes));
+    }
+
+    public HashedBytesRef(BytesRef bytes) {
+        this(bytes, bytes.hashCode());
+    }
+
+    public HashedBytesRef(BytesRef bytes, int hash) {
+        this.bytes = bytes;
+        this.hash = hash;
+    }
+
+    public HashedBytesRef resetHashCode() {
+        this.hash = bytes.hashCode();
+        return this;
+    }
+
+    public HashedBytesRef reset(BytesRef bytes, int hash) {
+        this.bytes = bytes;
+        this.hash = hash;
+        return this;
+    }
+
+    @Override
+    public int hashCode() {
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other instanceof HashedBytesRef) {
+            return bytes.equals(((HashedBytesRef) other).bytes);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return bytes.toString();
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java

@@ -0,0 +1,32 @@
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.settings.IndexSettings;
+
+/**
+ */
+public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
+
+    private final String fieldName;
+    protected final FieldDataType fieldDataType;
+    protected final IndexFieldDataCache cache;
+
+    public AbstractIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings);
+        this.fieldName = fieldName;
+        this.fieldDataType = fieldDataType;
+        this.cache = cache;
+    }
+
+    @Override
+    public String getFieldName() {
+        return this.fieldName;
+    }
+
+    @Override
+    public void clear() {
+        cache.clear(index, fieldName);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+/**
+ * The thread safe {@link org.apache.lucene.index.AtomicReader} level cache of the data.
+ */
+public interface AtomicFieldData<Script extends ScriptDocValues> {
+
+    /**
+     * Does *one* of the docs contain multi values?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Are the values ordered? (in ascending manner).
+     */
+    boolean isValuesOrdered();
+
+    /**
+     * The number of docs in this field data.
+     */
+    int getNumDocs();
+
+    /**
+     * Size (in bytes) of memory used by this field data.
+     */
+    long getMemorySizeInBytes();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    BytesValues getBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    HashedBytesValues getHashedBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as strings.
+     */
+    StringValues getStringValues();
+
+    /**
+     * Returns a "scripting" based values.
+     */
+    Script getScriptValues();
+}

src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+/**
+ */
+public interface AtomicNumericFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
+
+    ByteValues getByteValues();
+
+    ShortValues getShortValues();
+
+    IntValues getIntValues();
+
+    LongValues getLongValues();
+
+    FloatValues getFloatValues();
+
+    DoubleValues getDoubleValues();
+}

src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+/**
+ */
+public interface AtomicOrdinalFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    OrdinalsBytesValues getBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    OrdinalsHashedBytesValues getHashedBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as strings.
+     */
+    OrdinalsStringValues getStringValues();
+}

src/main/java/org/elasticsearch/index/fielddata/ByteValues.java

@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.ByteArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public interface ByteValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    byte getValue(int docId);
+
+    byte getValueMissing(int docId, byte missingValue);
+
+    ByteArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, byte value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        byte next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public byte next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public byte value;
+            public boolean done;
+
+            public Single reset(byte value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public byte next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class LongBased implements ByteValues {
+
+        private final LongValues values;
+
+        private final ByteArrayRef arrayScratch = new ByteArrayRef(new byte[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public byte getValue(int docId) {
+            return (byte) values.getValue(docId);
+        }
+
+        @Override
+        public byte getValueMissing(int docId, byte missingValue) {
+            return (byte) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public ByteArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return ByteArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (byte) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            public ValueIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public byte next() {
+                return (byte) iter.next();
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                proc.onValue(docId, (byte) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/BytesValues.java

@@ -0,0 +1,305 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public interface BytesValues {
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
+     */
+    BytesRef getValue(int docId);
+
+    /**
+     * Returns the bytes value for the docId, with the provided "ret" which will be filled with the
+     * result which will also be returned. If there is no value for this docId, the length will be 0.
+     * Note, the bytes are not "safe".
+     */
+    BytesRef getValueScratch(int docId, BytesRef ret);
+
+    /**
+     * Returns a bytes value for a docId. The content is guaranteed not to be shared.
+     */
+    BytesRef getValueSafe(int docId);
+
+    /**
+     * Returns an array wrapping all the bytes values for a doc. The content is guaranteed not to be shared.
+     */
+    BytesRefArrayRef getValues(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
+     */
+    Iter getIter(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
+     */
+    Iter getIterSafe(int docId);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
+
+    public static interface ValueInDocProc {
+        void onValue(int docId, BytesRef value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        BytesRef next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public BytesRef next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public BytesRef value;
+            public boolean done;
+
+            public Single reset(BytesRef value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public BytesRef next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class StringBased implements BytesValues {
+
+        private final StringValues values;
+
+        protected final BytesRef scratch = new BytesRef();
+        private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
+        private final ValueIter valueIter = new ValueIter();
+        private final SafeValueIter safeValueIter = new SafeValueIter();
+        private final Proc proc = new Proc();
+        private final SafeProc safeProc = new SafeProc();
+
+        public StringBased(StringValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public BytesRef getValue(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            scratch.copyChars(value);
+            return scratch;
+        }
+
+        @Override
+        public BytesRef getValueScratch(int docId, BytesRef ret) {
+            String value = values.getValue(docId);
+            if (value == null) {
+                ret.length = 0;
+                return ret;
+            }
+            ret.copyChars(value);
+            return ret;
+        }
+
+        @Override
+        public BytesRef getValueSafe(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            return new BytesRef(value);
+        }
+
+        @Override
+        public BytesRefArrayRef getValues(int docId) {
+            StringArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return BytesRefArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                String value = arrayRef.values[i];
+                arrayScratch.values[arrayScratch.end++] = value == null ? null : new BytesRef(value);
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public Iter getIterSafe(int docId) {
+            return safeValueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private final BytesRef scratch = new BytesRef();
+            private StringValues.Iter iter;
+
+            public ValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public BytesRef next() {
+                scratch.copyChars(iter.next());
+                return scratch;
+            }
+        }
+
+        static class SafeValueIter implements Iter {
+
+            private StringValues.Iter iter;
+
+            public SafeValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public BytesRef next() {
+                return new BytesRef(iter.next());
+            }
+        }
+
+        static class Proc implements StringValues.ValueInDocProc {
+
+            private final BytesRef scratch = new BytesRef();
+            private BytesValues.ValueInDocProc proc;
+
+            public Proc reset(BytesValues.ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                scratch.copyChars(value);
+                proc.onValue(docId, scratch);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+        static class SafeProc implements StringValues.ValueInDocProc {
+
+            private BytesValues.ValueInDocProc proc;
+
+            public SafeProc reset(BytesValues.ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                proc.onValue(docId, new BytesRef(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+
+/**
+ */
+public interface DoubleValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    double getValue(int docId);
+
+    double getValueMissing(int docId, double missingValue);
+
+    DoubleArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, double value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        double next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public double next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public double value;
+            public boolean done;
+
+            public Single reset(double value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public double next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.common.Nullable;
+
+/**
+ */
+public class FieldDataType {
+
+    private final String type;
+    @Nullable
+    private final String format;
+    private final ImmutableMap<String, String> options;
+
+    public FieldDataType(String type) {
+        this(type, null, ImmutableMap.<String, String>of());
+    }
+
+    public FieldDataType(String type, @Nullable String format, ImmutableMap<String, String> options) {
+        this.type = type;
+        this.format = format;
+        this.options = options;
+    }
+
+    public String getType() {
+        return this.type;
+    }
+
+    @Nullable
+    public String getFormat() {
+        return this.format;
+    }
+
+    public ImmutableMap<String, String> getOptions() {
+        return this.options;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/FloatValues.java

@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.FloatArrayRef;
+
+/**
+ */
+public interface FloatValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    float getValue(int docId);
+
+    float getValueMissing(int docId, float missingValue);
+
+    FloatArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, float value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        float next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public float next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public float value;
+            public boolean done;
+
+            public Single reset(float value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public float next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class DoubleBased implements FloatValues {
+
+        private final DoubleValues values;
+
+        private final FloatArrayRef arrayScratch = new FloatArrayRef(new float[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public DoubleBased(DoubleValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public float getValue(int docId) {
+            return (float) values.getValue(docId);
+        }
+
+        @Override
+        public float getValueMissing(int docId, float missingValue) {
+            return (float) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public FloatArrayRef getValues(int docId) {
+            DoubleArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return FloatArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (float) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private DoubleValues.Iter iter;
+
+            public ValueIter reset(DoubleValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public float next() {
+                return (float) iter.next();
+            }
+        }
+
+        static class Proc implements DoubleValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, double value) {
+                proc.onValue(docId, (float) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java

@@ -0,0 +1,411 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+
+/**
+ */
+public interface HashedBytesValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    /**
+     * Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
+     */
+    HashedBytesRef getValue(int docId);
+
+    /**
+     * Returns a bytes value for a docId. The content is guaranteed not to be shared.
+     */
+    HashedBytesRef getValueSafe(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
+     */
+    Iter getIter(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
+     */
+    Iter getIterSafe(int docId);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
+
+    public static interface ValueInDocProc {
+        void onValue(int docId, HashedBytesRef value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        HashedBytesRef next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public HashedBytesRef value;
+            public boolean done;
+
+            public Single reset(HashedBytesRef value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    /**
+     * A {@link BytesValues} based implementation.
+     */
+    static class BytesBased implements HashedBytesValues {
+
+        private final BytesValues values;
+
+        protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+        private final ValueIter valueIter = new ValueIter();
+        private final SafeValueIter safeValueIter = new SafeValueIter();
+        private final Proc proc = new Proc();
+        private final SafeProc safeProc = new SafeProc();
+
+        public BytesBased(BytesValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            BytesRef value = values.getValue(docId);
+            if (value == null) return null;
+            scratch.bytes = value;
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getValueSafe(int docId) {
+            return new HashedBytesRef(values.getValueSafe(docId));
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public Iter getIterSafe(int docId) {
+            return safeValueIter.reset(values.getIterSafe(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+            private BytesValues.Iter iter;
+
+            public ValueIter reset(BytesValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                scratch.bytes = iter.next();
+                return scratch.resetHashCode();
+            }
+        }
+
+        static class SafeValueIter implements Iter {
+
+            private BytesValues.Iter iter;
+
+            public SafeValueIter reset(BytesValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                return new HashedBytesRef(iter.next());
+            }
+        }
+
+        static class Proc implements BytesValues.ValueInDocProc {
+
+            private final HashedBytesRef scratch = new HashedBytesRef();
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, BytesRef value) {
+                scratch.bytes = value;
+                proc.onValue(docId, scratch.resetHashCode());
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+        static class SafeProc implements BytesValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public SafeProc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, BytesRef value) {
+                proc.onValue(docId, new HashedBytesRef(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+    }
+
+    static class StringBased implements HashedBytesValues {
+
+        private final StringValues values;
+
+        protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+        private final ValueIter valueIter = new ValueIter();
+        private final SafeValueIter safeValueIter = new SafeValueIter();
+        private final Proc proc = new Proc();
+        private final SafeProc safeProc = new SafeProc();
+
+        public StringBased(StringValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            scratch.bytes.copyChars(value);
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getValueSafe(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            return new HashedBytesRef(new BytesRef(values.getValue(docId)));
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public Iter getIterSafe(int docId) {
+            return safeValueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+            private StringValues.Iter iter;
+
+            public ValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                scratch.bytes.copyChars(iter.next());
+                return scratch.resetHashCode();
+            }
+        }
+
+        static class SafeValueIter implements Iter {
+
+            private StringValues.Iter iter;
+
+            public SafeValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                return new HashedBytesRef(new BytesRef(iter.next()));
+            }
+        }
+
+        static class Proc implements StringValues.ValueInDocProc {
+
+            private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                scratch.bytes.copyChars(value);
+                proc.onValue(docId, scratch);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+        static class SafeProc implements StringValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public SafeProc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                proc.onValue(docId, new HashedBytesRef(new BytesRef(value)));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexComponent;
+import org.elasticsearch.index.settings.IndexSettings;
+
+/**
+ */
+public interface IndexFieldData<FD extends AtomicFieldData> extends IndexComponent {
+
+    /**
+     * The field name.
+     */
+    String getFieldName();
+
+    /**
+     * Are the values ordered? (in ascending manner).
+     */
+    boolean valuesOrdered();
+
+    /**
+     * Loads the atomic field data for the reader, possibly cached.
+     */
+    FD load(AtomicReaderContext context);
+
+    /**
+     * Loads directly the atomic field data for the reader, ignoring any caching involved.
+     */
+    FD loadDirect(AtomicReaderContext context) throws Exception;
+
+    /**
+     * Comparator used for sorting.
+     */
+    XFieldComparatorSource comparatorSource(@Nullable Object missingValue);
+
+    /**
+     * Clears any resources associated with this field data.
+     */
+    void clear();
+
+    // we need this extended source we we have custom comparators to reuse our field data
+    // in this case, we need to reduce type that will be used when search results are reduced
+    // on another node (we don't have the custom source them...)
+    public abstract class XFieldComparatorSource extends FieldComparatorSource {
+
+        public abstract SortField.Type reducedType();
+    }
+
+    interface Builder {
+
+        IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.SegmentReader;
+import org.elasticsearch.index.Index;
+
+import java.util.concurrent.Callable;
+
+/**
+ * A simple field data cache abstraction.
+ */
+public interface IndexFieldDataCache {
+
+    <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(AtomicReaderContext context, IFD indexFieldData) throws Exception;
+
+    void clear(Index index);
+
+    void clear(Index index, String fieldName);
+
+    /**
+     * The resident field data cache is a *per field* cache that keeps all the values in memory.
+     */
+    static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener {
+        private final Cache<Object, AtomicFieldData> cache;
+
+        protected FieldBased(Cache<Object, AtomicFieldData> cache) {
+            this.cache = cache;
+        }
+
+        @Override
+        public void onClose(SegmentReader owner) {
+            cache.invalidate(owner.getCoreCacheKey());
+        }
+
+        @Override
+        public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
+            //noinspection unchecked
+            return (FD) cache.get(context.reader().getCoreCacheKey(), new Callable<AtomicFieldData>() {
+                @Override
+                public AtomicFieldData call() throws Exception {
+                    return indexFieldData.loadDirect(context);
+                }
+            });
+        }
+
+        @Override
+        public void clear(Index index) {
+            cache.invalidateAll();
+        }
+
+        @Override
+        public void clear(Index index, String fieldName) {
+            cache.invalidateAll();
+        }
+    }
+
+    static class Resident extends FieldBased {
+
+        public Resident() {
+            super(CacheBuilder.newBuilder().<Object, AtomicFieldData>build());
+        }
+    }
+
+    static class Soft extends FieldBased {
+
+        public Soft() {
+            super(CacheBuilder.newBuilder().softValues().<Object, AtomicFieldData>build());
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java

@@ -0,0 +1,124 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.plain.ConcreteBytesRefIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ */
+public class IndexFieldDataService extends AbstractIndexComponent {
+
+    private final static ImmutableMap<String, IndexFieldData.Builder> buildersByType;
+    private final static ImmutableMap<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
+
+    static {
+        buildersByType = MapBuilder.<String, IndexFieldData.Builder>newMapBuilder()
+                .put("string", new ConcreteBytesRefIndexFieldData.Builder())
+                .put("double", new DoubleArrayIndexFieldData.Builder())
+                .immutableMap();
+
+        buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder().
+                put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
+                .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
+                .immutableMap();
+    }
+
+    private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
+
+    public IndexFieldDataService(Index index) {
+        this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
+    }
+
+    @Inject
+    public IndexFieldDataService(Index index, @IndexSettings Settings indexSettings) {
+        super(index, indexSettings);
+    }
+
+    public void clear() {
+        synchronized (loadedFieldData) {
+            for (IndexFieldData fieldData : loadedFieldData.values()) {
+                fieldData.clear();
+            }
+            loadedFieldData.clear();
+        }
+    }
+
+    public void clearField(String fieldName) {
+        synchronized (loadedFieldData) {
+            IndexFieldData fieldData = loadedFieldData.remove(fieldName);
+            if (fieldData != null) {
+                fieldData.clear();
+            }
+        }
+    }
+
+    public <IFD extends IndexFieldData> IFD getForField(String fieldName, FieldDataType type) {
+        IndexFieldData fieldData = loadedFieldData.get(type.getType());
+        if (fieldData == null) {
+            synchronized (loadedFieldData) {
+                fieldData = loadedFieldData.get(type.getType());
+                if (fieldData == null) {
+                    IndexFieldData.Builder builder = null;
+                    if (type.getFormat() != null) {
+                        builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), type.getFormat()));
+                    }
+                    if (builder == null) {
+                        builder = buildersByType.get(type.getType());
+                    }
+                    if (builder == null) {
+                        throw new ElasticSearchIllegalArgumentException("failed to find field data builder for field " + fieldName + ", and type " + type);
+                    }
+
+                    IndexFieldDataCache cache;
+                    if (type.getOptions().containsKey("cache")) {
+                        String cacheType = type.getOptions().get("cache");
+                        if ("resident".equals(cacheType)) {
+                            cache = new IndexFieldDataCache.Resident();
+                        } else if ("soft".equals(cacheType)) {
+                            cache = new IndexFieldDataCache.Soft();
+                        } else {
+                            throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]");
+                        }
+                    } else {
+                        cache = new IndexFieldDataCache.Resident();
+                    }
+
+                    fieldData = builder.build(index, indexSettings, fieldName, type, cache);
+                    loadedFieldData.put(fieldName, fieldData);
+                }
+            }
+        }
+        return (IFD) fieldData;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.AtomicReaderContext;
+
+/**
+ */
+public interface IndexNumericFieldData<FD extends AtomicNumericFieldData> extends IndexFieldData<FD> {
+
+    static enum NumericType {
+        BYTE,
+        SHORT,
+        INT,
+        LONG,
+        FLOAT,
+        DOUBLE
+    }
+
+    NumericType getNumericType();
+
+    /**
+     * Loads the atomic field data for the reader, possibly cached.
+     */
+    FD load(AtomicReaderContext context);
+
+    /**
+     * Loads directly the atomic field data for the reader, ignoring any caching involved.
+     */
+    FD loadDirect(AtomicReaderContext context) throws Exception;
+}

src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalFieldData.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.AtomicReaderContext;
+
+/**
+ */
+public interface IndexOrdinalFieldData<FD extends AtomicOrdinalFieldData> extends IndexFieldData<FD> {
+
+    /**
+     * Loads the atomic field data for the reader, possibly cached.
+     */
+    FD load(AtomicReaderContext context);
+
+    /**
+     * Loads directly the atomic field data for the reader, ignoring any caching involved.
+     */
+    FD loadDirect(AtomicReaderContext context) throws Exception;
+}

src/main/java/org/elasticsearch/index/fielddata/IntValues.java

@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public interface IntValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    int getValue(int docId);
+
+    int getValueMissing(int docId, int missingValue);
+
+    IntArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, int value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        int next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public int next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public int value;
+            public boolean done;
+
+            public Single reset(int value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public int next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class LongBased implements IntValues {
+
+        private final LongValues values;
+
+        private final IntArrayRef arrayScratch = new IntArrayRef(new int[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public int getValue(int docId) {
+            return (int) values.getValue(docId);
+        }
+
+        @Override
+        public int getValueMissing(int docId, int missingValue) {
+            return (int) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public IntArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return IntArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (int) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            public ValueIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public int next() {
+                return (int) iter.next();
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                proc.onValue(docId, (int) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/LongValues.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public interface LongValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    long getValue(int docId);
+
+    long getValueMissing(int docId, long missingValue);
+
+    LongArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, long value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        long next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public long next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public long value;
+            public boolean done;
+
+            public Single reset(long value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public long next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+/**
+ */
+public interface OrdinalsBytesValues extends BytesValues {
+
+    Ordinals.Docs ordinals();
+
+    BytesRef getValueByOrd(int ord);
+
+    /**
+     * Returns the bytes value for the docId, with the provided "ret" which will be filled with the
+     * result which will also be returned. If there is no value for this docId, the length will be 0.
+     * Note, the bytes are not "safe".
+     */
+    BytesRef getValueScratchByOrd(int ord, BytesRef ret);
+
+    BytesRef getSafeValueByOrd(int ord);
+
+    public static class StringBased extends BytesValues.StringBased implements OrdinalsBytesValues {
+
+        private final OrdinalsStringValues values;
+
+        public StringBased(OrdinalsStringValues values) {
+            super(values);
+            this.values = values;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return values.ordinals();
+        }
+
+        @Override
+        public BytesRef getValueByOrd(int ord) {
+            scratch.copyChars(values.getValueByOrd(ord));
+            return scratch;
+        }
+
+        @Override
+        public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
+            ret.copyChars(values.getValueByOrd(ord));
+            return ret;
+        }
+
+        @Override
+        public BytesRef getSafeValueByOrd(int ord) {
+            return new BytesRef(values.getValueByOrd(ord));
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+/**
+ */
+public interface OrdinalsHashedBytesValues extends HashedBytesValues {
+
+    Ordinals.Docs ordinals();
+
+    HashedBytesRef getValueByOrd(int ord);
+
+    HashedBytesRef getSafeValueByOrd(int ord);
+
+    static class BytesBased extends HashedBytesValues.BytesBased implements OrdinalsHashedBytesValues {
+
+        private final OrdinalsBytesValues values;
+
+        public BytesBased(OrdinalsBytesValues values) {
+            super(values);
+            this.values = values;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return values.ordinals();
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            scratch.bytes = values.getValueByOrd(ord);
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return new HashedBytesRef(values.getSafeValueByOrd(ord));
+        }
+    }
+
+    static class StringBased extends HashedBytesValues.StringBased implements OrdinalsHashedBytesValues {
+
+        private final OrdinalsStringValues values;
+
+        public StringBased(OrdinalsStringValues values) {
+            super(values);
+            this.values = values;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return values.ordinals();
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            scratch.bytes.copyChars(values.getValueByOrd(ord));
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return new HashedBytesRef(new BytesRef(values.getValueByOrd(ord)));
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+/**
+ */
+public interface OrdinalsStringValues extends StringValues {
+
+    Ordinals.Docs ordinals();
+
+    String getValueByOrd(int ord);
+}

src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java

@@ -0,0 +1,258 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.util.*;
+
+/**
+ * Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
+ * and a <code>getValues</code> that return the relevant type that then can be used in scripts.
+ */
+public interface ScriptDocValues {
+
+    void setNextDocId(int docId);
+
+    boolean isEmpty();
+
+    static class Strings implements ScriptDocValues {
+
+        private final StringValues values;
+        private int docId;
+
+        public Strings(StringValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public String getValue() {
+            return values.getValue(docId);
+        }
+
+        public StringArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class Bytes implements ScriptDocValues {
+
+        private final BytesValues values;
+        private int docId;
+
+        public Bytes(BytesValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public BytesRef getValue() {
+            return values.getValue(docId);
+        }
+
+        public BytesRefArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericByte implements ScriptDocValues {
+
+        private final ByteValues values;
+        private int docId;
+
+        public NumericByte(ByteValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public byte getValue() {
+            return values.getValue(docId);
+        }
+
+        public ByteArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericShort implements ScriptDocValues {
+
+        private final ShortValues values;
+        private int docId;
+
+        public NumericShort(ShortValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public short getValue() {
+            return values.getValue(docId);
+        }
+
+        public ShortArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericInteger implements ScriptDocValues {
+
+        private final IntValues values;
+        private int docId;
+
+        public NumericInteger(IntValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public int getValue() {
+            return values.getValue(docId);
+        }
+
+        public IntArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericLong implements ScriptDocValues {
+
+        private final LongValues values;
+        private int docId;
+
+        public NumericLong(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public long getValue() {
+            return values.getValue(docId);
+        }
+
+        public LongArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericFloat implements ScriptDocValues {
+
+        private final FloatValues values;
+        private int docId;
+
+        public NumericFloat(FloatValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public float getValue() {
+            return values.getValue(docId);
+        }
+
+        public FloatArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericDouble implements ScriptDocValues {
+
+        private final DoubleValues values;
+        private int docId;
+
+        public NumericDouble(DoubleValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public double getValue() {
+            return values.getValue(docId);
+        }
+
+        public DoubleArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/ShortValues.java

@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+import org.elasticsearch.index.fielddata.util.ShortArrayRef;
+
+/**
+ */
+public interface ShortValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    short getValue(int docId);
+
+    short getValueMissing(int docId, short missingValue);
+
+    ShortArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, short value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        short next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public short next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public short value;
+            public boolean done;
+
+            public Single reset(short value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public short next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class LongBased implements ShortValues {
+
+        private final LongValues values;
+
+        private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public short getValue(int docId) {
+            return (short) values.getValue(docId);
+        }
+
+        @Override
+        public short getValueMissing(int docId, short missingValue) {
+            return (short) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public ShortArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return ShortArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (short) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            public ValueIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public short next() {
+                return (short) iter.next();
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                proc.onValue(docId, (short) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/StringValues.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public interface StringValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    String getValue(int docId);
+
+    StringArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    /**
+     * Go over all the possible values.
+     */
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    public static interface ValueInDocProc {
+        void onValue(int docId, String value);
+
+        void onMissing(int docId);
+    }
+
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        String next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public String next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public String value;
+            public boolean done;
+
+            public Single reset(String value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public String next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexFieldData indexFieldData;
+
+    public BytesRefFieldComparatorSource(IndexFieldData indexFieldData) {
+        this.indexFieldData = indexFieldData;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.STRING;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldName());
+        if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexOrdinalFieldData) {
+            return new BytesRefOrdValComparator((IndexOrdinalFieldData) indexFieldData, numHits);
+        }
+        return new BytesRefValComparator(indexFieldData, numHits);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java

@@ -0,0 +1,475 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
+import org.elasticsearch.index.fielddata.OrdinalsBytesValues;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+import java.io.IOException;
+
+/**
+ * Sorts by field's natural Term sort order, using
+ * ordinals.  This is functionally equivalent to {@link
+ * org.apache.lucene.search.FieldComparator.TermValComparator}, but it first resolves the string
+ * to their relative ordinal positions (using the index
+ * returned by {@link org.apache.lucene.search.FieldCache#getTermsIndex}), and
+ * does most comparisons using the ordinals.  For medium
+ * to large results, this comparator will be much faster
+ * than {@link org.apache.lucene.search.FieldComparator.TermValComparator}.  For very small
+ * result sets it may be slower.
+ */
+public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
+
+    final IndexOrdinalFieldData indexFieldData;
+
+    /* Ords for each slot.
+       @lucene.internal */
+    final int[] ords;
+
+    /* Values for each slot.
+       @lucene.internal */
+    final BytesRef[] values;
+
+    /* Which reader last copied a value into the slot. When
+       we compare two slots, we just compare-by-ord if the
+       readerGen is the same; else we must compare the
+       values (slower).
+       @lucene.internal */
+    final int[] readerGen;
+
+    /* Gen of current reader we are on.
+       @lucene.internal */
+    int currentReaderGen = -1;
+
+    /* Current reader's doc ord/values.
+       @lucene.internal */
+    OrdinalsBytesValues termsIndex;
+
+    /* Bottom slot, or -1 if queue isn't full yet
+       @lucene.internal */
+    int bottomSlot = -1;
+
+    /* Bottom ord (same as ords[bottomSlot] once bottomSlot
+       is set).  Cached for faster compares.
+       @lucene.internal */
+    int bottomOrd;
+
+    /* True if current bottom slot matches the current
+       reader.
+       @lucene.internal */
+    boolean bottomSameReader;
+
+    /* Bottom value (same as values[bottomSlot] once
+       bottomSlot is set).  Cached for faster compares.
+      @lucene.internal */
+    BytesRef bottomValue;
+
+    final BytesRef tempBR = new BytesRef();
+
+    public BytesRefOrdValComparator(IndexOrdinalFieldData indexFieldData, int numHits) {
+        this.indexFieldData = indexFieldData;
+        ords = new int[numHits];
+        values = new BytesRef[numHits];
+        readerGen = new int[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        if (readerGen[slot1] == readerGen[slot2]) {
+            return ords[slot1] - ords[slot2];
+        }
+
+        final BytesRef val1 = values[slot1];
+        final BytesRef val2 = values[slot2];
+        if (val1 == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+        return val1.compareTo(val2);
+    }
+
+    @Override
+    public int compareBottom(int doc) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void copy(int slot, int doc) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int compareDocToValue(int doc, BytesRef value) {
+        BytesRef docValue = termsIndex.getValue(doc);
+        if (docValue == null) {
+            if (value == null) {
+                return 0;
+            }
+            return -1;
+        } else if (value == null) {
+            return 1;
+        }
+        return docValue.compareTo(value);
+    }
+
+    /**
+     * Base class for specialized (per bit width of the
+     * ords) per-segment comparator.  NOTE: this is messy;
+     * we do this only because hotspot can't reliably inline
+     * the underlying array access when looking up doc->ord
+     *
+     * @lucene.internal
+     */
+    abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
+
+        @Override
+        public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+            return BytesRefOrdValComparator.this.setNextReader(context);
+        }
+
+        @Override
+        public int compare(int slot1, int slot2) {
+            return BytesRefOrdValComparator.this.compare(slot1, slot2);
+        }
+
+        @Override
+        public void setBottom(final int bottom) {
+            BytesRefOrdValComparator.this.setBottom(bottom);
+        }
+
+        @Override
+        public BytesRef value(int slot) {
+            return BytesRefOrdValComparator.this.value(slot);
+        }
+
+        @Override
+        public int compareValues(BytesRef val1, BytesRef val2) {
+            if (val1 == null) {
+                if (val2 == null) {
+                    return 0;
+                }
+                return -1;
+            } else if (val2 == null) {
+                return 1;
+            }
+            return val1.compareTo(val2);
+        }
+
+        @Override
+        public int compareDocToValue(int doc, BytesRef value) {
+            return BytesRefOrdValComparator.this.compareDocToValue(doc, value);
+        }
+    }
+
+    // Used per-segment when bit width of doc->ord is 8:
+    private final class ByteOrdComparator extends PerSegmentComparator {
+        private final byte[] readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public ByteOrdComparator(byte[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
+            this.readerOrds = readerOrds;
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = (readerOrds[doc] & 0xFF);
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds[doc] & 0xFF;
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    // Used per-segment when bit width of doc->ord is 16:
+    private final class ShortOrdComparator extends PerSegmentComparator {
+        private final short[] readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public ShortOrdComparator(short[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
+            this.readerOrds = readerOrds;
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = (readerOrds[doc] & 0xFFFF);
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds[doc] & 0xFFFF;
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    // Used per-segment when bit width of doc->ord is 32:
+    private final class IntOrdComparator extends PerSegmentComparator {
+        private final int[] readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public IntOrdComparator(int[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
+            this.readerOrds = readerOrds;
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = readerOrds[doc];
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds[doc];
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    // Used per-segment when bit width is not a native array
+    // size (8, 16, 32):
+    final class AnyOrdComparator extends PerSegmentComparator {
+        private final IndexFieldData fieldData;
+        private final Ordinals.Docs readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public AnyOrdComparator(IndexFieldData fieldData, OrdinalsBytesValues termsIndex, int docBase) {
+            this.fieldData = fieldData;
+            this.readerOrds = termsIndex.ordinals();
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = readerOrds.getOrd(doc);
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds.getOrd(doc);
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    @Override
+    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+        final int docBase = context.docBase;
+        termsIndex = indexFieldData.load(context).getBytesValues();
+        // TODO, we should support sorting on multi valued field, take the best ascending value out of all the values
+        if (termsIndex.isMultiValued()) {
+            throw new ElasticSearchIllegalArgumentException("can't sort on a multi valued field");
+        }
+        final Ordinals.Docs docToOrd = termsIndex.ordinals();
+        Object ordsStorage = docToOrd.ordinals().getBackingStorage();
+        FieldComparator<BytesRef> perSegComp = null;
+
+        if (ordsStorage instanceof byte[]) {
+            perSegComp = new ByteOrdComparator((byte[]) ordsStorage, termsIndex, docBase);
+        } else if (ordsStorage instanceof short[]) {
+            perSegComp = new ShortOrdComparator((short[]) ordsStorage, termsIndex, docBase);
+        } else if (ordsStorage instanceof int[]) {
+            perSegComp = new IntOrdComparator((int[]) ordsStorage, termsIndex, docBase);
+        }
+        // Don't specialize the long[] case since it's not
+        // possible, ie, worse case is MAX_INT-1 docs with
+        // every one having a unique value.
+
+        // TODO: ES - should we optimize for the PackedInts.Reader case as well?
+        if (perSegComp == null) {
+            perSegComp = new AnyOrdComparator(indexFieldData, termsIndex, docBase);
+        }
+
+        currentReaderGen++;
+        if (bottomSlot != -1) {
+            perSegComp.setBottom(bottomSlot);
+        }
+
+        return perSegComp;
+    }
+
+    @Override
+    public void setBottom(final int bottom) {
+        bottomSlot = bottom;
+
+        bottomValue = values[bottomSlot];
+        if (currentReaderGen == readerGen[bottomSlot]) {
+            bottomOrd = ords[bottomSlot];
+            bottomSameReader = true;
+        } else {
+            if (bottomValue == null) {
+                // 0 ord is null for all segments
+                assert ords[bottomSlot] == 0;
+                bottomOrd = 0;
+                bottomSameReader = true;
+                readerGen[bottomSlot] = currentReaderGen;
+            } else {
+                final int index = binarySearch(termsIndex, bottomValue);
+                if (index < 0) {
+                    bottomOrd = -index - 2;
+                    bottomSameReader = false;
+                } else {
+                    bottomOrd = index;
+                    // exact value match
+                    bottomSameReader = true;
+                    readerGen[bottomSlot] = currentReaderGen;
+                    ords[bottomSlot] = bottomOrd;
+                }
+            }
+        }
+    }
+
+    @Override
+    public BytesRef value(int slot) {
+        return values[slot];
+    }
+
+    final protected static int binarySearch(OrdinalsBytesValues a, BytesRef key) {
+        return binarySearch(a, key, 1, a.ordinals().getNumDocs() - 1);
+    }
+
+    final protected static int binarySearch(OrdinalsBytesValues a, BytesRef key, int low, int high) {
+
+        while (low <= high) {
+            int mid = (low + high) >>> 1;
+            BytesRef midVal = a.getValueByOrd(mid);
+            int cmp;
+            if (midVal != null) {
+                cmp = midVal.compareTo(key);
+            } else {
+                cmp = -1;
+            }
+
+            if (cmp < 0)
+                low = mid + 1;
+            else if (cmp > 0)
+                high = mid - 1;
+            else
+                return mid;
+        }
+        return -(low + 1);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java

@@ -0,0 +1,119 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.BytesValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+
+import java.io.IOException;
+
+/**
+ * Sorts by field's natural Term sort order.  All
+ * comparisons are done using BytesRef.compareTo, which is
+ * slow for medium to large result sets but possibly
+ * very fast for very small results sets.
+ */
+public final class BytesRefValComparator extends FieldComparator<BytesRef> {
+
+    private final IndexFieldData indexFieldData;
+    private BytesRef[] values;
+    private BytesValues docTerms;
+    private BytesRef bottom;
+
+    BytesRefValComparator(IndexFieldData indexFieldData, int numHits) {
+        values = new BytesRef[numHits];
+        this.indexFieldData = indexFieldData;
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final BytesRef val1 = values[slot1];
+        final BytesRef val2 = values[slot2];
+        if (val1 == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+
+        return val1.compareTo(val2);
+    }
+
+    @Override
+    public int compareBottom(int doc) {
+        BytesRef val2 = docTerms.getValue(doc);
+        if (bottom == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+        return bottom.compareTo(val2);
+    }
+
+    @Override
+    public void copy(int slot, int doc) {
+        if (values[slot] == null) {
+            values[slot] = new BytesRef();
+        }
+        docTerms.getValueScratch(doc, values[slot]);
+    }
+
+    @Override
+    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+        docTerms = indexFieldData.load(context).getBytesValues();
+        return this;
+    }
+
+    @Override
+    public void setBottom(final int bottom) {
+        this.bottom = values[bottom];
+    }
+
+    @Override
+    public BytesRef value(int slot) {
+        return values[slot];
+    }
+
+    @Override
+    public int compareValues(BytesRef val1, BytesRef val2) {
+        if (val1 == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+        return val1.compareTo(val2);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, BytesRef value) {
+        return docTerms.getValue(doc).compareTo(value);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class DoubleValuesComparator extends FieldComparator<Double> {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final double missingValue;
+
+    protected final double[] values;
+    private double bottom;
+    private DoubleValues readerValues;
+
+    public DoubleValuesComparator(IndexNumericFieldData indexFieldData, double missingValue, int numHits) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+        this.values = new double[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final double v1 = values[slot1];
+        final double v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void setBottom(int slot) {
+        this.bottom = values[slot];
+    }
+
+    @Override
+    public int compareBottom(int doc) throws IOException {
+        double v2 = readerValues.getValueMissing(doc, missingValue);
+
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void copy(int slot, int doc) throws IOException {
+        values[slot] = readerValues.getValueMissing(doc, missingValue);
+    }
+
+    @Override
+    public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getDoubleValues();
+        return this;
+    }
+
+    @Override
+    public Double value(int slot) {
+        return Double.valueOf(values[slot]);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Double valueObj) throws IOException {
+        final double value = valueObj.doubleValue();
+        double docValue = readerValues.getValueMissing(doc, missingValue);
+        if (docValue < value) {
+            return -1;
+        } else if (docValue > value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final Object missingValue;
+
+    public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.DOUBLE;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldName());
+
+        double dMissingValue;
+        if (missingValue == null || "_last".equals(missingValue)) {
+            dMissingValue = reversed ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
+        } else if ("_first".equals(missingValue)) {
+            dMissingValue = reversed ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
+        } else {
+            dMissingValue = missingValue instanceof Number ? ((Number) missingValue).doubleValue() : Double.parseDouble(missingValue.toString());
+        }
+
+        return new DoubleValuesComparator(indexFieldData, dMissingValue, numHits);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java

@@ -0,0 +1,122 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ * Ordinals that effectively are single valued and map "one to one" to the
+ * doc ids. Note, the docId is incremented by 1 to get the ordinal, since 0
+ * denotes an empty value.
+ */
+public class DocIdOrdinals implements Ordinals {
+
+    private final int numDocs;
+
+    /**
+     * Constructs a new doc id ordinals.
+     */
+    public DocIdOrdinals(int numDocs) {
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return null;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        return RamUsage.NUM_BYTES_OBJECT_REF;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numDocs + 1;
+    }
+
+    @Override
+    public Ordinals.Docs ordinals() {
+        return new Docs(this);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final DocIdOrdinals parent;
+        private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
+        private final SingleValueIter iter = new SingleValueIter();
+
+        public Docs(DocIdOrdinals parent) {
+            this.parent = parent;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return docId + 1;
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            intsScratch.values[0] = docId + 1;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(docId + 1);
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, docId + 1);
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ */
+public class EmptyOrdinals implements Ordinals {
+
+    private final int numDocs;
+
+    public EmptyOrdinals(int numDocs) {
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        return 0;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return null;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return this.numDocs;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numDocs + 1;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final EmptyOrdinals parent;
+
+        public Docs(EmptyOrdinals parent) {
+            this.parent = parent;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return 0;
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            return IntArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return EmptyIter.INSTANCE;
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, 0);
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiFlatArrayOrdinals.java

@@ -0,0 +1,187 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ * "Flat" multi valued ordinals, the first level array size is as the maximum
+ * values a docId has. Ordinals are populated in order from the first flat array
+ * value to the next.
+ */
+public class MultiFlatArrayOrdinals implements Ordinals {
+
+    private ThreadLocal<IntArrayRef> intArrayRefCache = new ThreadLocal<IntArrayRef>() {
+        @Override
+        protected IntArrayRef initialValue() {
+            return new IntArrayRef(new int[ordinals.length]);
+        }
+    };
+
+    // ordinals with value 0 indicates no value
+    private final int[][] ordinals;
+    private final int numDocs;
+    private final int numOrds;
+
+    private long size = -1;
+
+    public MultiFlatArrayOrdinals(int[][] ordinals, int numOrds) {
+        assert ordinals.length > 0;
+        this.ordinals = ordinals;
+        this.numDocs = ordinals[0].length;
+        this.numOrds = numOrds;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return ordinals;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            long size = 0;
+            size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
+            for (int[] ordinal : ordinals) {
+                size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
+            }
+            this.size = size;
+        }
+        return size;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return true;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numOrds;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this, ordinals, intArrayRefCache.get());
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final MultiFlatArrayOrdinals parent;
+        private final int[][] ordinals;
+        private final IterImpl iter;
+
+        private final IntArrayRef intsScratch;
+
+        public Docs(MultiFlatArrayOrdinals parent, int[][] ordinals, IntArrayRef intsScratch) {
+            this.parent = parent;
+            this.ordinals = ordinals;
+            this.iter = new IterImpl(ordinals);
+            this.intsScratch = intsScratch;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return this.parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return true;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return ordinals[0][docId];
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            intsScratch.end = 0;
+            int i;
+            for (i = 0; i < ordinals.length; i++) {
+                int ordinal = ordinals[i][docId];
+                if (ordinal == 0) {
+                    if (i == 0) return IntArrayRef.EMPTY;
+                    break;
+                }
+                intsScratch.values[i] = ordinal;
+            }
+            intsScratch.end = i;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(docId);
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            for (int i = 0; i < ordinals.length; i++) {
+                int ordinal = ordinals[i][docId];
+                if (ordinal == 0) {
+                    if (i == 0) proc.onOrdinal(docId, 0);
+                    return;
+                }
+                proc.onOrdinal(docId, ordinal);
+            }
+        }
+
+        public static class IterImpl implements Docs.Iter {
+
+            private final int[][] ordinals;
+            private int docId;
+            private int i;
+
+            public IterImpl(int[][] ordinals) {
+                this.ordinals = ordinals;
+            }
+
+            public IterImpl reset(int docId) {
+                this.docId = docId;
+                this.i = 0;
+                return this;
+            }
+
+            @Override
+            public int next() {
+                if (i >= ordinals.length) return 0;
+                return ordinals[i++][docId];
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java

@@ -0,0 +1,153 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ * A thread safe ordinals abstraction. Ordinals can only be positive integers.
+ */
+public interface Ordinals {
+
+    /**
+     * Returns the backing storage for this ordinals.
+     */
+    Object getBackingStorage();
+
+    /**
+     * The memory size this ordinals take.
+     */
+    long getMemorySizeInBytes();
+
+    /**
+     * Is one of the docs maps to more than one ordinal?
+     */
+    boolean isMultiValued();
+
+    /**
+     * The number of docs in this ordinals.
+     */
+    int getNumDocs();
+
+    /**
+     * The number of ordinals.
+     */
+    int getNumOrds();
+
+    /**
+     * Returns a lightweight (non thread safe) view iterator of the ordinals.
+     */
+    Docs ordinals();
+
+    /**
+     * A non thread safe ordinals abstraction, yet very lightweight to create. The idea
+     * is that this gets created for each "iteration" over ordinals.
+     * <p/>
+     * <p>A value of 0 ordinal when iterating indicated "no" value.</p>
+     */
+    interface Docs {
+
+        /**
+         * Returns the original ordinals used to generate this Docs "itereator".
+         */
+        Ordinals ordinals();
+
+        /**
+         * The number of docs in this ordinals.
+         */
+        int getNumDocs();
+
+        /**
+         * The number of ordinals.
+         */
+        int getNumOrds();
+
+        /**
+         * Is one of the docs maps to more than one ordinal?
+         */
+        boolean isMultiValued();
+
+        /**
+         * The ordinal that maps to the relevant docId. If it has no value, returns
+         * <tt>0</tt>.
+         */
+        int getOrd(int docId);
+
+        /**
+         * Returns an array of ordinals matching the docIds, with 0 length one
+         * for a doc with no ordinals.
+         */
+        IntArrayRef getOrds(int docId);
+
+        /**
+         * Returns an iterator of the ordinals that match the docId, with an
+         * empty iterator for a doc with no ordinals.
+         */
+        Iter getIter(int docId);
+
+        /**
+         * Iterates over the ordinals associated with a docId. If there are no values,
+         * a callback with a value 0 will be done.
+         */
+        void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc);
+
+        public static interface OrdinalInDocProc {
+            void onOrdinal(int docId, int ordinal);
+        }
+
+        /**
+         * An iterator over ordinals values.
+         */
+        interface Iter {
+
+            /**
+             * Gets the next ordinal. Returning 0 if the iteration is exhausted.
+             */
+            int next();
+        }
+
+        static class EmptyIter implements Iter {
+
+            public static EmptyIter INSTANCE = new EmptyIter();
+
+            @Override
+            public int next() {
+                return 0;
+            }
+        }
+
+        static class SingleValueIter implements Iter {
+
+            private int value;
+
+            public SingleValueIter reset(int value) {
+                this.value = value;
+                return this;
+            }
+
+            @Override
+            public int next() {
+                int actual = value;
+                value = 0;
+                return actual;
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ */
+public class SingleArrayOrdinals implements Ordinals {
+
+    // ordinals with value 0 indicates no value
+    private final int[] ordinals;
+    private final int numOrds;
+
+    private long size = -1;
+
+    public SingleArrayOrdinals(int[] ordinals, int numOrds) {
+        this.ordinals = ordinals;
+        this.numOrds = numOrds;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return ordinals;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            size = RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
+        }
+        return size;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return ordinals.length;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numOrds;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this, ordinals);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final SingleArrayOrdinals parent;
+        private final int[] ordinals;
+
+        private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
+        private final SingleValueIter iter = new SingleValueIter();
+
+        public Docs(SingleArrayOrdinals parent, int[] ordinals) {
+            this.parent = parent;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return ordinals[docId];
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            int ordinal = ordinals[docId];
+            if (ordinal == 0) return IntArrayRef.EMPTY;
+            intsScratch.values[0] = docId;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(ordinals[docId]);
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, ordinals[docId]);
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.apache.lucene.util.packed.PackedInts;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ */
+public class SinglePackedOrdinals implements Ordinals {
+
+    // ordinals with value 0 indicates no value
+    private final PackedInts.Reader reader;
+    private final int numOrds;
+
+    private long size = -1;
+
+    public SinglePackedOrdinals(PackedInts.Reader reader, int numOrds) {
+        this.reader = reader;
+        this.numOrds = numOrds;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        if (reader.hasArray()) {
+            return reader.getArray();
+        }
+        return reader;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            size = RamUsage.NUM_BYTES_OBJECT_REF + reader.ramBytesUsed();
+        }
+        return size;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return reader.size();
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numOrds;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this, reader);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final SinglePackedOrdinals parent;
+        private final PackedInts.Reader reader;
+
+        private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
+        private final SingleValueIter iter = new SingleValueIter();
+
+        public Docs(SinglePackedOrdinals parent, PackedInts.Reader reader) {
+            this.parent = parent;
+            this.reader = reader;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return (int) reader.get(docId);
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            int ordinal = (int) reader.get(docId);
+            if (ordinal == 0) return IntArrayRef.EMPTY;
+            intsScratch.values[0] = docId;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset((int) reader.get(docId));
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, (int) reader.get(docId));
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java

@@ -0,0 +1,673 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
+
+    // 0 ordinal in values means no value (its null)
+    private final BytesRef[] values;
+    private final Ordinals ordinals;
+
+    private int[] hashes;
+    private long size = -1;
+
+    public ConcreteBytesRefAtomicFieldData(BytesRef[] values, Ordinals ordinals) {
+        this.values = values;
+        this.ordinals = ordinals;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return ordinals.isMultiValued();
+    }
+
+    @Override
+    public int getNumDocs() {
+        return ordinals.getNumDocs();
+    }
+
+    @Override
+    public boolean isValuesOrdered() {
+        return true;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            long size = RamUsage.NUM_BYTES_ARRAY_HEADER;
+            for (BytesRef value : values) {
+                if (value != null) {
+                    size += RamUsage.NUM_BYTES_OBJECT_REF + RamUsage.NUM_BYTES_OBJECT_HEADER +
+                            RamUsage.NUM_BYTES_ARRAY_HEADER + (value.length + (2 * RamUsage.NUM_BYTES_INT));
+                }
+            }
+            size += ordinals.getMemorySizeInBytes();
+            this.size = size;
+        }
+        return size;
+    }
+
+    @Override
+    public OrdinalsBytesValues getBytesValues() {
+        return ordinals.isMultiValued() ? new BytesValues.Multi(values, ordinals.ordinals()) : new BytesValues.Single(values, ordinals.ordinals());
+    }
+
+    @Override
+    public OrdinalsHashedBytesValues getHashedBytesValues() {
+        if (hashes == null) {
+            int[] hashes = new int[values.length];
+            for (int i = 0; i < values.length; i++) {
+                BytesRef value = values[i];
+                hashes[i] = value == null ? 0 : value.hashCode();
+            }
+            this.hashes = hashes;
+        }
+        return ordinals.isMultiValued() ? new HashedBytesValues.Multi(values, hashes, ordinals.ordinals()) : new HashedBytesValues.Single(values, hashes, ordinals.ordinals());
+    }
+
+    @Override
+    public OrdinalsStringValues getStringValues() {
+        return ordinals.isMultiValued() ? new StringValues.Multi(values, ordinals.ordinals()) : new StringValues.Single(values, ordinals.ordinals());
+    }
+
+    @Override
+    public ScriptDocValues.Strings getScriptValues() {
+        return new ScriptDocValues.Strings(getStringValues());
+    }
+
+    static abstract class BytesValues implements org.elasticsearch.index.fielddata.OrdinalsBytesValues {
+
+        protected final BytesRef[] values;
+        protected final Ordinals.Docs ordinals;
+
+        BytesValues(BytesRef[] values, Ordinals.Docs ordinals) {
+            this.values = values;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return this.ordinals;
+        }
+
+        @Override
+        public BytesRef getValueByOrd(int ord) {
+            return values[ord];
+        }
+
+        @Override
+        public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
+            BytesRef value = values[ord];
+            if (value == null) {
+                ret.length = 0;
+            } else {
+                ret.bytes = value.bytes;
+                ret.offset = value.offset;
+                ret.length = value.offset;
+            }
+            return ret;
+        }
+
+        @Override
+        public BytesRef getSafeValueByOrd(int ord) {
+            return values[ord];
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public BytesRef getValue(int docId) {
+            return values[ordinals.getOrd(docId)];
+        }
+
+        @Override
+        public BytesRef getValueScratch(int docId, BytesRef ret) {
+            BytesRef value = values[ordinals.getOrd(docId)];
+            if (value == null) {
+                ret.length = 0;
+            } else {
+                ret.bytes = value.bytes;
+                ret.offset = value.offset;
+                ret.length = value.length;
+            }
+            return ret;
+        }
+
+        @Override
+        public BytesRef getValueSafe(int docId) {
+            return values[ordinals.getOrd(docId)];
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+            forEachValueInDoc(docId, proc);
+        }
+
+        static class Single extends BytesValues {
+
+            private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public BytesRefArrayRef getValues(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return BytesRefArrayRef.EMPTY;
+                arrayScratch.values[0] = values[ord];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(values[ord]);
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(values[ord]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, values[ord]);
+                }
+            }
+        }
+
+        static class Multi extends BytesValues {
+
+            private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[10], 0);
+            private final ValuesIter iter;
+
+            Multi(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public BytesRefArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return BytesRefArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final BytesRef[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(BytesRef[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public BytesRef next() {
+                    BytesRef value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    static abstract class HashedBytesValues implements org.elasticsearch.index.fielddata.OrdinalsHashedBytesValues {
+
+        protected final BytesRef[] values;
+        protected final int[] hashes;
+        protected final Ordinals.Docs ordinals;
+
+        protected final HashedBytesRef scratch = new HashedBytesRef();
+
+        HashedBytesValues(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
+            this.values = values;
+            this.hashes = hashes;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return this.ordinals;
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            return scratch.reset(values[ord], hashes[ord]);
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return new HashedBytesRef(values[ord], hashes[ord]);
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return scratch.reset(values[ord], hashes[ord]);
+        }
+
+        @Override
+        public HashedBytesRef getValueSafe(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return new HashedBytesRef(values[ord], hashes[ord]);
+        }
+
+        static class Single extends HashedBytesValues {
+
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
+                super(values, hashes, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(scratch.reset(values[ord], hashes[ord]));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(new HashedBytesRef(values[ord], hashes[ord]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
+                }
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
+                }
+            }
+        }
+
+        static class Multi extends HashedBytesValues {
+
+            private final ValuesIter iter;
+            private final SafeValuesIter safeIter;
+
+            Multi(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
+                super(values, hashes, ordinals);
+                this.iter = new ValuesIter(values, hashes);
+                this.safeIter = new SafeValuesIter(values, hashes);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return safeIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final BytesRef[] values;
+                private final int[] hashes;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                private final HashedBytesRef scratch = new HashedBytesRef();
+
+                ValuesIter(BytesRef[] values, int[] hashes) {
+                    this.values = values;
+                    this.hashes = hashes;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public HashedBytesRef next() {
+                    HashedBytesRef value = scratch.reset(values[ord], hashes[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+
+            static class SafeValuesIter implements Iter {
+
+                private final BytesRef[] values;
+                private final int[] hashes;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                SafeValuesIter(BytesRef[] values, int[] hashes) {
+                    this.values = values;
+                    this.hashes = hashes;
+                }
+
+                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public HashedBytesRef next() {
+                    HashedBytesRef value = new HashedBytesRef(values[ord], hashes[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    static abstract class StringValues implements OrdinalsStringValues {
+
+        protected final BytesRef[] values;
+        protected final Ordinals.Docs ordinals;
+
+        protected StringValues(BytesRef[] values, Ordinals.Docs ordinals) {
+            this.values = values;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return ordinals;
+        }
+
+        @Override
+        public String getValueByOrd(int ord) {
+            BytesRef value = values[ord];
+            if (value == null) return null;
+            return value.utf8ToString();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public String getValue(int docId) {
+            BytesRef value = values[ordinals.getOrd(docId)];
+            if (value == null) return null;
+            return value.utf8ToString();
+        }
+
+        static class Single extends StringValues {
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return StringArrayRef.EMPTY;
+                BytesRef value = values[ord];
+                arrayScratch.values[0] = value == null ? null : value.utf8ToString();
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(values[ord].utf8ToString());
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                proc.onValue(docId, values[ord].utf8ToString());
+            }
+        }
+
+        static class Multi extends StringValues {
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[10], 0);
+            private final ValuesIter iter;
+
+            Multi(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+                iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return StringArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    BytesRef value = values[ords.values[i]];
+                    arrayScratch.values[arrayScratch.end++] = value == null ? null : value.utf8ToString();
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    BytesRef value = values[ord];
+                    proc.onValue(docId, value == null ? null : value.utf8ToString());
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements StringValues.Iter {
+
+                private final BytesRef[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(BytesRef[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public String next() {
+                    BytesRef value = values[ord];
+                    ord = ordsIter.next();
+                    return value == null ? null : value.utf8ToString();
+                }
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java

@@ -0,0 +1,155 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class ConcreteBytesRefIndexFieldData extends AbstractIndexFieldData<ConcreteBytesRefAtomicFieldData> implements IndexOrdinalFieldData<ConcreteBytesRefAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
+            return new ConcreteBytesRefIndexFieldData(index, indexSettings, fieldName, type, cache);
+        }
+    }
+
+    public ConcreteBytesRefIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldName, fieldDataType, cache);
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        return true;
+    }
+
+    @Override
+    public ConcreteBytesRefAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public ConcreteBytesRefAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldName());
+        if (terms == null) {
+            return new ConcreteBytesRefAtomicFieldData(new BytesRef[1], new EmptyOrdinals(reader.maxDoc()));
+        }
+
+        long size = terms.size();
+        if (size == -1) {
+            size = 1024;
+        }
+        final ArrayList<BytesRef> values = new ArrayList<BytesRef>((int) size);
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add(null); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add(BytesRef.deepCopyOf(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            // optimizing to DocIdOrdinals which will use less memory removes the "order" notion of ordinals
+            // we can potentially do this, but only for things like facet processes
+            // that don't require order
+//            int[] nativeOrdinals = ordinals.get(0);
+//            boolean allHaveValue = true;
+//            for (int nativeOrdinal : nativeOrdinals) {
+//                if (nativeOrdinal == 0) {
+//                    allHaveValue = false;
+//                    break;
+//                }
+//            }
+//            if (allHaveValue) {
+//                BytesRef[] convertedValues = new BytesRef[values.size()];
+//                for (int i = 1; i < nativeOrdinals.length; i++) {
+//                    convertedValues[i] = values.get(nativeOrdinals[i]);
+//                }
+//                return new ConcreteBytesRefAtomicFieldData(convertedValues, DocIdOrdinals.INSTANCE);
+//            } else {
+//                return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new SingleArrayOrdinals(nativeOrdinals));
+//            }
+            return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new SingleArrayOrdinals(ordinals.get(0), termOrd));
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        // TODO support "missingValue" for sortMissingValue options here...
+        return new BytesRefFieldComparatorSource(this);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java

@@ -0,0 +1,913 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldData {
+
+    protected final double[] values;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public DoubleArrayAtomicFieldData(double[] values, int numDocs) {
+        this.values = values;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    public static class WithOrdinals extends DoubleArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(double[] values, int numDocs, Ordinals ordinals) {
+            super(values, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + +RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, ordinals.ordinals());
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final ValuesIter valuesIter;
+
+            StringValues(double[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.valuesIter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public String getValue(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return null;
+                }
+                return Double.toString(values[ord]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return StringArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = Double.toString(values[ords.values[i]]);
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return valuesIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, Double.toString(values[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final double[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public String next() {
+                    String value = Double.toString(values[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final double[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final ValuesIter iter;
+
+            LongValues(double[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (long) values[ord];
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return LongArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (long) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements LongValues.Iter {
+
+                private final double[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public long next() {
+                    double value = values[ord];
+                    ord = ordsIter.next();
+                    return (long) value;
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final double[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final ValuesIter iter;
+
+            DoubleValues(double[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return values[ord];
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return DoubleArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements DoubleValues.Iter {
+
+                private final double[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public double next() {
+                    double value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    /**
+     * A single valued case, where not all values are "set", so we have a FixedBitSet that
+     * indicates which values have an actual value.
+     */
+    public static class SingleFixedSet extends DoubleArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(double[] values, int numDocs, FixedBitSet set) {
+            super(values, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values, set);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, set);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, set);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] values;
+            private final FixedBitSet set;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(double[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public String getValue(int docId) {
+                if (set.get(docId)) {
+                    return Double.toString(values[docId]);
+                } else {
+                    return null;
+                }
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = Double.toString(values[docId]);
+                    return arrayScratch;
+                } else {
+                    return StringArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(Double.toString(values[docId]));
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, Double.toString(values[docId]));
+                } else {
+                    proc.onMissing(docId);
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final double[] values;
+            private final FixedBitSet set;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(double[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                if (set.get(docId)) {
+                    return (long) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (long) values[docId];
+                    return arrayScratch;
+                } else {
+                    return LongArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((long) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (long) values[docId]);
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final double[] values;
+            private final FixedBitSet set;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(double[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                if (set.get(docId)) {
+                    return values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = values[docId];
+                    return arrayScratch;
+                } else {
+                    return DoubleArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, values[docId]);
+                }
+            }
+        }
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends DoubleArrayAtomicFieldData {
+
+        /**
+         * Note, here, we assume that there is no offset by 1 from docId, so position 0
+         * is the value for docId 0.
+         */
+        public Single(double[] values, int numDocs) {
+            super(values, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] values;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(double[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public String getValue(int docId) {
+                return Double.toString(values[docId]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                arrayScratch.values[0] = Double.toString(values[docId]);
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(Double.toString(values[docId]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, Double.toString(values[docId]));
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final double[] values;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(double[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (long) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((long) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (long) values[docId]);
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final double[] values;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(double[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                return values[docId];
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                arrayScratch.values[0] = values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, values[docId]);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java

@@ -0,0 +1,156 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TDoubleArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArrayAtomicFieldData> implements IndexNumericFieldData<DoubleArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
+            return new DoubleArrayIndexFieldData(index, indexSettings, fieldName, type, cache);
+        }
+    }
+
+    public DoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldName, fieldDataType, cache);
+    }
+
+    @Override
+    public NumericType getNumericType() {
+        return NumericType.DOUBLE;
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public DoubleArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public DoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldName());
+        if (terms == null) {
+            return new DoubleArrayAtomicFieldData.Single(new double[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TDoubleArrayList values = new TDoubleArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add(0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add(FieldCache.NUMERIC_UTILS_DOUBLE_PARSER.parseDouble(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            double[] sValues = new double[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sValues[i] = values.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new DoubleArrayAtomicFieldData.Single(sValues, reader.maxDoc());
+            } else {
+                return new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new DoubleArrayAtomicFieldData.WithOrdinals(values.toArray(new double[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        return new DoubleValuesComparatorSource(this, missingValue);
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/ByteArrayRef.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Bytes;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class ByteArrayRef extends AbstractList<Byte> implements RandomAccess {
+
+    public static final ByteArrayRef EMPTY = new ByteArrayRef(new byte[0]);
+
+    public byte[] values;
+    public int start;
+    public int end;
+
+    public ByteArrayRef(byte[] values) {
+        this(values, 0, values.length);
+    }
+
+    public ByteArrayRef(byte[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public ByteArrayRef(byte[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new byte[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Byte get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Byte)
+                && indexOf(values, (Byte) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Byte) {
+            int i = indexOf(values, (Byte) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Byte) {
+            int i = lastIndexOf(values, (Byte) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Byte set(int index, Byte element) {
+        assert index < size();
+        byte oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof ByteArrayRef) {
+            ByteArrayRef that = (ByteArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Bytes.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(byte[] array, byte target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(byte[] array, byte target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/BytesRefArrayRef.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ */
+public class BytesRefArrayRef {
+    public static final BytesRefArrayRef EMPTY = new BytesRefArrayRef(new BytesRef[0]);
+
+    public BytesRef[] values;
+    public int start;
+    public int end;
+
+    public BytesRefArrayRef(BytesRef[] values) {
+        this(values, 0, values.length);
+    }
+
+    public BytesRefArrayRef(BytesRef[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public BytesRefArrayRef(BytesRef[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new BytesRef[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    public int size() {
+        return end - start;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/DoubleArrayRef.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Doubles;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class DoubleArrayRef extends AbstractList<Double> implements RandomAccess {
+
+    public static final DoubleArrayRef EMPTY = new DoubleArrayRef(new double[0]);
+
+    public double[] values;
+    public int start;
+    public int end;
+
+    public DoubleArrayRef(double[] values) {
+        this(values, 0, values.length);
+    }
+
+    public DoubleArrayRef(double[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public DoubleArrayRef(double[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new double[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Double get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Double)
+                && indexOf(values, (Double) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Double) {
+            int i = indexOf(values, (Double) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Double) {
+            int i = lastIndexOf(values, (Double) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Double set(int index, Double element) {
+        assert index < size();
+        double oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof DoubleArrayRef) {
+            DoubleArrayRef that = (DoubleArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Doubles.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(double[] array, double target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(double[] array, double target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/FloatArrayRef.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Floats;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class FloatArrayRef extends AbstractList<Float> implements RandomAccess {
+
+    public static final FloatArrayRef EMPTY = new FloatArrayRef(new float[0]);
+
+    public float[] values;
+    public int start;
+    public int end;
+
+    public FloatArrayRef(float[] values) {
+        this(values, 0, values.length);
+    }
+
+    public FloatArrayRef(float[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public FloatArrayRef(float[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new float[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Float get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Float)
+                && indexOf(values, (Float) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Float) {
+            int i = indexOf(values, (Float) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Float) {
+            int i = lastIndexOf(values, (Float) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Float set(int index, Float element) {
+        assert index < size();
+        float oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof FloatArrayRef) {
+            FloatArrayRef that = (FloatArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Floats.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(float[] array, float target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(float[] array, float target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/IntArrayRef.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Ints;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class IntArrayRef extends AbstractList<Integer> implements RandomAccess {
+
+    public static final IntArrayRef EMPTY = new IntArrayRef(new int[0]);
+
+    public int[] values;
+    public int start;
+    public int end;
+
+    public IntArrayRef(int[] values) {
+        this(values, 0, values.length);
+    }
+
+    public IntArrayRef(int[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public IntArrayRef(int[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new int[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Integer get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Integer)
+                && indexOf(values, (Integer) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Integer) {
+            int i = indexOf(values, (Integer) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Integer) {
+            int i = lastIndexOf(values, (Integer) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Integer set(int index, Integer element) {
+        assert index < size();
+        int oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof IntArrayRef) {
+            IntArrayRef that = (IntArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Ints.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(int[] array, int target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(int[] array, int target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/LongArrayRef.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Longs;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class LongArrayRef extends AbstractList<Long> implements RandomAccess {
+
+    public static final LongArrayRef EMPTY = new LongArrayRef(new long[0]);
+
+    public long[] values;
+    public int start;
+    public int end;
+
+    public LongArrayRef(long[] values) {
+        this(values, 0, values.length);
+    }
+
+    public LongArrayRef(long[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public LongArrayRef(long[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new long[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Long get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Long)
+                && indexOf(values, (Long) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Long) {
+            int i = indexOf(values, (Long) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Long) {
+            int i = lastIndexOf(values, (Long) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Long set(int index, Long element) {
+        assert index < size();
+        long oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof LongArrayRef) {
+            LongArrayRef that = (LongArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Longs.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(long[] array, long target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(long[] array, long target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/ShortArrayRef.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Shorts;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class ShortArrayRef extends AbstractList<Short> implements RandomAccess {
+
+    public static final ShortArrayRef EMPTY = new ShortArrayRef(new short[0]);
+
+    public short[] values;
+    public int start;
+    public int end;
+
+    public ShortArrayRef(short[] values) {
+        this(values, 0, values.length);
+    }
+
+    public ShortArrayRef(short[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public ShortArrayRef(short[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new short[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Short get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Short)
+                && indexOf(values, (Short) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Short) {
+            int i = indexOf(values, (Short) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Short) {
+            int i = lastIndexOf(values, (Short) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Short set(int index, Short element) {
+        assert index < size();
+        short oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof ShortArrayRef) {
+            ShortArrayRef that = (ShortArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Shorts.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(short[] array, short target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(short[] array, short target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}

src/main/java/org/elasticsearch/index/fielddata/util/StringArrayRef.java

@@ -0,0 +1,149 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class StringArrayRef extends AbstractList<String> implements RandomAccess {
+
+    public static final StringArrayRef EMPTY = new StringArrayRef(new String[0]);
+
+    public String[] values;
+    public int start;
+    public int end;
+
+    public StringArrayRef(String[] values) {
+        this(values, 0, values.length);
+    }
+
+    public StringArrayRef(String[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public StringArrayRef(String[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new String[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public String get(int index) {
+        assert index >= 0 && index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        String sTarget = target.toString();
+        for (int i = start; i < end; i++) {
+            if (values[i].equals(sTarget)) return true;
+        }
+        return false;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        String sTarget = target.toString();
+        for (int i = start; i < end; i++) {
+            if (values[i].equals(sTarget)) return (i - start);
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        String sTarget = target.toString();
+        for (int i = end - 1; i >= start; i--) {
+            if (values[i].equals(sTarget)) return (i - start);
+        }
+        return -1;
+    }
+
+    @Override
+    public String set(int index, String element) {
+        assert index >= 0 && index < size();
+        String oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof StringArrayRef) {
+            StringArrayRef that = (StringArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (!values[start + i].equals(that.values[that.start + i])) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + values[i].hashCode();
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+}

src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java

@@ -0,0 +1,413 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+
+/**
+ */
+@Test
+public abstract class AbstractFieldDataTests {
+
+    protected IndexFieldDataService ifdService;
+    protected IndexWriter writer;
+    protected AtomicReaderContext readerContext;
+
+    protected abstract FieldDataType getFieldDataType();
+
+    public <IFD extends IndexFieldData> IFD getForField(String fieldName) {
+        return ifdService.getForField(fieldName, getFieldDataType());
+    }
+
+    @BeforeMethod
+    public void setup() throws Exception {
+        ifdService = new IndexFieldDataService(new Index("test"));
+        writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));
+    }
+
+    protected AtomicReaderContext refreshReader() throws Exception {
+        if (readerContext != null) {
+            readerContext.reader().close();
+        }
+        AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(writer, true));
+        readerContext = reader.getContext();
+        return readerContext;
+    }
+
+    @AfterMethod
+    public void tearDown() throws Exception {
+        if (readerContext != null) {
+            readerContext.reader().close();
+        }
+        writer.close();
+        ifdService.clear();
+    }
+
+    public static class BytesValuesVerifierProc implements BytesValues.ValueInDocProc {
+
+        private static final BytesRef MISSING = new BytesRef();
+
+        private final int docId;
+        private final List<BytesRef> expected = new ArrayList<BytesRef>();
+
+        private int idx;
+
+        BytesValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public BytesValuesVerifierProc addExpected(String value) {
+            expected.add(new BytesRef(value));
+            return this;
+        }
+
+        public BytesValuesVerifierProc addExpected(BytesRef value) {
+            expected.add(value);
+            return this;
+        }
+
+        public BytesValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, BytesRef value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class HashedBytesValuesVerifierProc implements HashedBytesValues.ValueInDocProc {
+
+        private static final HashedBytesRef MISSING = new HashedBytesRef();
+
+        private final int docId;
+        private final List<HashedBytesRef> expected = new ArrayList<HashedBytesRef>();
+
+        private int idx;
+
+        HashedBytesValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public HashedBytesValuesVerifierProc addExpected(String value) {
+            expected.add(new HashedBytesRef(value));
+            return this;
+        }
+
+        public HashedBytesValuesVerifierProc addExpected(BytesRef value) {
+            expected.add(new HashedBytesRef(value));
+            return this;
+        }
+
+        public HashedBytesValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, HashedBytesRef value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class StringValuesVerifierProc implements StringValues.ValueInDocProc {
+
+        private static final String MISSING = new String();
+
+        private final int docId;
+        private final List<String> expected = new ArrayList<String>();
+
+        private int idx;
+
+        StringValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public StringValuesVerifierProc addExpected(String value) {
+            expected.add(value);
+            return this;
+        }
+
+        public StringValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, String value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class LongValuesVerifierProc implements LongValues.ValueInDocProc {
+
+        private static final Long MISSING = new Long(0);
+
+        private final int docId;
+        private final List<Long> expected = new ArrayList<Long>();
+
+        private int idx;
+
+        LongValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public LongValuesVerifierProc addExpected(long value) {
+            expected.add(value);
+            return this;
+        }
+
+        public LongValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, long value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class DoubleValuesVerifierProc implements DoubleValues.ValueInDocProc {
+
+        private static final Double MISSING = new Double(0);
+
+        private final int docId;
+        private final List<Double> expected = new ArrayList<Double>();
+
+        private int idx;
+
+        DoubleValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public DoubleValuesVerifierProc addExpected(double value) {
+            expected.add(value);
+            return this;
+        }
+
+        public DoubleValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, double value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class ByteValuesVerifierProc implements ByteValues.ValueInDocProc {
+
+        private static final Byte MISSING = new Byte((byte) 0);
+
+        private final int docId;
+        private final List<Byte> expected = new ArrayList<Byte>();
+
+        private int idx;
+
+        ByteValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public ByteValuesVerifierProc addExpected(byte value) {
+            expected.add(value);
+            return this;
+        }
+
+        public ByteValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, byte value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class ShortValuesVerifierProc implements ShortValues.ValueInDocProc {
+
+        private static final Short MISSING = new Short((byte) 0);
+
+        private final int docId;
+        private final List<Short> expected = new ArrayList<Short>();
+
+        private int idx;
+
+        ShortValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public ShortValuesVerifierProc addExpected(short value) {
+            expected.add(value);
+            return this;
+        }
+
+        public ShortValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, short value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class IntValuesVerifierProc implements IntValues.ValueInDocProc {
+
+        private static final Integer MISSING = new Integer(0);
+
+        private final int docId;
+        private final List<Integer> expected = new ArrayList<Integer>();
+
+        private int idx;
+
+        IntValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public IntValuesVerifierProc addExpected(int value) {
+            expected.add(value);
+            return this;
+        }
+
+        public IntValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, int value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class FloatValuesVerifierProc implements FloatValues.ValueInDocProc {
+
+        private static final Float MISSING = new Float(0);
+
+        private final int docId;
+        private final List<Float> expected = new ArrayList<Float>();
+
+        private int idx;
+
+        FloatValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public FloatValuesVerifierProc addExpected(float value) {
+            expected.add(value);
+            return this;
+        }
+
+        public FloatValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, float value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+}

src/test/java/org/elasticsearch/test/unit/index/fielddata/ConcreteBytesStringFieldDataTests.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.index.fielddata.FieldDataType;
+import org.testng.annotations.Test;
+
+/**
+ */
+@Test
+public class ConcreteBytesStringFieldDataTests extends StringFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("string", "concrete_bytes", ImmutableMap.<String, String>of());
+    }
+}

src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.elasticsearch.index.fielddata.FieldDataType;
+
+/**
+ */
+public class DoubleFieldDataTests extends NumericFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("double");
+    }
+}

src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java

@@ -0,0 +1,741 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ */
+@Test
+public abstract class StringFieldDataTests extends AbstractFieldDataTests {
+
+    protected String one() {
+        return "1";
+    }
+
+    protected String two() {
+        return "2";
+    }
+
+    protected String three() {
+        return "3";
+    }
+
+    protected String four() {
+        return "4";
+    }
+
+    protected void fillSingleValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "1", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testSingleValueAllSet() throws Exception {
+        fillSingleValueAllSet();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicReaderContext readerContext = refreshReader();
+        AtomicFieldData fieldData = indexFieldData.load(readerContext);
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData.getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(false));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(true));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
+        assertThat(bytesRef, equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(true));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), equalTo(one()));
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(one()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+
+        IndexSearcher searcher = new IndexSearcher(readerContext.reader());
+        TopFieldDocs topDocs;
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null))));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null), true)));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
+    }
+
+    protected void fillSingleValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testSingleValueWithMissing() throws Exception {
+        fillSingleValueWithMissing();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData
+                .getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(false));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(false));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), nullValue());
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), nullValue());
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
+        assertThat(bytesRef, equalTo(new BytesRef()));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(0));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIter(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), nullValue());
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), nullValue());
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIter(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(false));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), nullValue());
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(0));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+
+        // TODO properly support missing....
+    }
+
+    protected void fillMultiValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        d.add(new StringField("value", "4", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new StringField("value", "1", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testMultiValueAllSet() throws Exception {
+        fillMultiValueAllSet();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData.getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(true));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(true));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
+        assertThat(bytesRef, equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(2));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(true));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), equalTo(one()));
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(2));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+        assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(four()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(one()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+    }
+
+    protected void fillMultiValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        d.add(new StringField("value", "4", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testMultiValueWithMissing() throws Exception {
+        fillMultiValueWithMissing();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData.getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(true));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(false));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), nullValue());
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), nullValue());
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
+        assertThat(bytesRef, equalTo(new BytesRef()));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(2));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(0));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIter(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), nullValue());
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), nullValue());
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIter(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(false));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), nullValue());
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(2));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+        assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(0));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(four()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+    }
+}