From 7397007e05ed361805ecc3c2752e1472062c918f Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Fri, 18 Jan 2013 12:00:49 +0100
Subject: [PATCH 01/35] initial commit

---
 .../common/lucene/HashedBytesRef.java         |   76 +
 .../fielddata/AbstractIndexFieldData.java     |   32 +
 .../index/fielddata/AtomicFieldData.java      |   66 +
 .../fielddata/AtomicNumericFieldData.java     |   37 +
 .../fielddata/AtomicOrdinalFieldData.java     |   40 +
 .../index/fielddata/ByteValues.java           |  198 +++
 .../index/fielddata/BytesValues.java          |  305 ++++
 .../index/fielddata/DoubleValues.java         |  100 ++
 .../index/fielddata/FieldDataType.java        |   56 +
 .../index/fielddata/FloatValues.java          |  198 +++
 .../index/fielddata/HashedBytesValues.java    |  411 +++++
 .../index/fielddata/IndexFieldData.java       |   77 +
 .../index/fielddata/IndexFieldDataCache.java  |   91 ++
 .../fielddata/IndexFieldDataService.java      |  124 ++
 .../fielddata/IndexNumericFieldData.java      |   48 +
 .../fielddata/IndexOrdinalFieldData.java      |   37 +
 .../index/fielddata/IntValues.java            |  198 +++
 .../index/fielddata/LongValues.java           |  100 ++
 .../index/fielddata/OrdinalsBytesValues.java  |   73 +
 .../fielddata/OrdinalsHashedBytesValues.java  |   87 ++
 .../index/fielddata/OrdinalsStringValues.java |   31 +
 .../index/fielddata/ScriptDocValues.java      |  258 ++++
 .../index/fielddata/ShortValues.java          |  198 +++
 .../index/fielddata/StringValues.java         |  102 ++
 .../BytesRefFieldComparatorSource.java        |   52 +
 .../BytesRefOrdValComparator.java             |  475 ++++++
 .../BytesRefValComparator.java                |  119 ++
 .../DoubleValuesComparator.java               |  105 ++
 .../DoubleValuesComparatorSource.java         |   62 +
 .../fielddata/ordinals/DocIdOrdinals.java     |  122 ++
 .../fielddata/ordinals/EmptyOrdinals.java     |  112 ++
 .../ordinals/MultiFlatArrayOrdinals.java      |  187 +++
 .../index/fielddata/ordinals/Ordinals.java    |  153 ++
 .../ordinals/SingleArrayOrdinals.java         |  129 ++
 .../ordinals/SinglePackedOrdinals.java        |  133 ++
 .../ConcreteBytesRefAtomicFieldData.java      |  673 +++++++++
 .../plain/ConcreteBytesRefIndexFieldData.java |  155 ++
 .../plain/DoubleArrayAtomicFieldData.java     |  913 +++++++++++
 .../plain/DoubleArrayIndexFieldData.java      |  156 ++
 .../index/fielddata/util/ByteArrayRef.java    |  172 +++
 .../fielddata/util/BytesRefArrayRef.java      |   59 +
 .../index/fielddata/util/DoubleArrayRef.java  |  172 +++
 .../index/fielddata/util/FloatArrayRef.java   |  172 +++
 .../index/fielddata/util/IntArrayRef.java     |  172 +++
 .../index/fielddata/util/LongArrayRef.java    |  172 +++
 .../index/fielddata/util/ShortArrayRef.java   |  172 +++
 .../index/fielddata/util/StringArrayRef.java  |  149 ++
 .../fielddata/AbstractFieldDataTests.java     |  413 +++++
 .../ConcreteBytesStringFieldDataTests.java    |   35 +
 .../index/fielddata/DoubleFieldDataTests.java |   32 +
 .../fielddata/NumericFieldDataTests.java      | 1344 +++++++++++++++++
 .../index/fielddata/StringFieldDataTests.java |  741 +++++++++
 52 files changed, 10294 insertions(+)
 create mode 100644 src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/FloatValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IntValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/LongValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/StringValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiFlatArrayOrdinals.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/ByteArrayRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/BytesRefArrayRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/DoubleArrayRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/FloatArrayRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/IntArrayRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/LongArrayRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/ShortArrayRef.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/StringArrayRef.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/ConcreteBytesStringFieldDataTests.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java b/src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java
new file mode 100644
index 00000000000..299de543e9a
--- /dev/null
+++ b/src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.lucene;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A wrapped to {@link BytesRef} that also caches the hashCode for it.
+ */
+public class HashedBytesRef {
+
+    public BytesRef bytes;
+    public int hash;
+
+    public HashedBytesRef() {
+    }
+
+    public HashedBytesRef(String bytes) {
+        this(new BytesRef(bytes));
+    }
+
+    public HashedBytesRef(BytesRef bytes) {
+        this(bytes, bytes.hashCode());
+    }
+
+    public HashedBytesRef(BytesRef bytes, int hash) {
+        this.bytes = bytes;
+        this.hash = hash;
+    }
+
+    public HashedBytesRef resetHashCode() {
+        this.hash = bytes.hashCode();
+        return this;
+    }
+
+    public HashedBytesRef reset(BytesRef bytes, int hash) {
+        this.bytes = bytes;
+        this.hash = hash;
+        return this;
+    }
+
+    @Override
+    public int hashCode() {
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other instanceof HashedBytesRef) {
+            return bytes.equals(((HashedBytesRef) other).bytes);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return bytes.toString();
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
new file mode 100644
index 00000000000..894d98f50fa
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
@@ -0,0 +1,32 @@
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.settings.IndexSettings;
+
+/**
+ */
+public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
+
+    private final String fieldName;
+    protected final FieldDataType fieldDataType;
+    protected final IndexFieldDataCache cache;
+
+    public AbstractIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings);
+        this.fieldName = fieldName;
+        this.fieldDataType = fieldDataType;
+        this.cache = cache;
+    }
+
+    @Override
+    public String getFieldName() {
+        return this.fieldName;
+    }
+
+    @Override
+    public void clear() {
+        cache.clear(index, fieldName);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
new file mode 100644
index 00000000000..1648a320397
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+/**
+ * The thread safe {@link org.apache.lucene.index.AtomicReader} level cache of the data.
+ */
+public interface AtomicFieldData<Script extends ScriptDocValues> {
+
+    /**
+     * Does *one* of the docs contain multi values?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Are the values ordered? (in ascending manner).
+     */
+    boolean isValuesOrdered();
+
+    /**
+     * The number of docs in this field data.
+     */
+    int getNumDocs();
+
+    /**
+     * Size (in bytes) of memory used by this field data.
+     */
+    long getMemorySizeInBytes();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    BytesValues getBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    HashedBytesValues getHashedBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as strings.
+     */
+    StringValues getStringValues();
+
+    /**
+     * Returns a "scripting" based values.
+     */
+    Script getScriptValues();
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
new file mode 100644
index 00000000000..748c7e07925
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+/**
+ */
+public interface AtomicNumericFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
+
+    ByteValues getByteValues();
+
+    ShortValues getShortValues();
+
+    IntValues getIntValues();
+
+    LongValues getLongValues();
+
+    FloatValues getFloatValues();
+
+    DoubleValues getDoubleValues();
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java
new file mode 100644
index 00000000000..7258a8da47c
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+/**
+ */
+public interface AtomicOrdinalFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    OrdinalsBytesValues getBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as bytes.
+     */
+    OrdinalsHashedBytesValues getHashedBytesValues();
+
+    /**
+     * Use a non thread safe (lightweight) view of the values as strings.
+     */
+    OrdinalsStringValues getStringValues();
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java b/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
new file mode 100644
index 00000000000..86d4e885936
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.ByteArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public interface ByteValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    byte getValue(int docId);
+
+    byte getValueMissing(int docId, byte missingValue);
+
+    ByteArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, byte value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        byte next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public byte next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public byte value;
+            public boolean done;
+
+            public Single reset(byte value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public byte next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class LongBased implements ByteValues {
+
+        private final LongValues values;
+
+        private final ByteArrayRef arrayScratch = new ByteArrayRef(new byte[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public byte getValue(int docId) {
+            return (byte) values.getValue(docId);
+        }
+
+        @Override
+        public byte getValueMissing(int docId, byte missingValue) {
+            return (byte) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public ByteArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return ByteArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (byte) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            public ValueIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public byte next() {
+                return (byte) iter.next();
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                proc.onValue(docId, (byte) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
new file mode 100644
index 00000000000..9b4ade8ea61
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public interface BytesValues {
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
+     */
+    BytesRef getValue(int docId);
+
+    /**
+     * Returns the bytes value for the docId, with the provided "ret" which will be filled with the
+     * result which will also be returned. If there is no value for this docId, the length will be 0.
+     * Note, the bytes are not "safe".
+     */
+    BytesRef getValueScratch(int docId, BytesRef ret);
+
+    /**
+     * Returns a bytes value for a docId. The content is guaranteed not to be shared.
+     */
+    BytesRef getValueSafe(int docId);
+
+    /**
+     * Returns an array wrapping all the bytes values for a doc. The content is guaranteed not to be shared.
+     */
+    BytesRefArrayRef getValues(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
+     */
+    Iter getIter(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
+     */
+    Iter getIterSafe(int docId);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
+
+    public static interface ValueInDocProc {
+        void onValue(int docId, BytesRef value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        BytesRef next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public BytesRef next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public BytesRef value;
+            public boolean done;
+
+            public Single reset(BytesRef value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public BytesRef next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class StringBased implements BytesValues {
+
+        private final StringValues values;
+
+        protected final BytesRef scratch = new BytesRef();
+        private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
+        private final ValueIter valueIter = new ValueIter();
+        private final SafeValueIter safeValueIter = new SafeValueIter();
+        private final Proc proc = new Proc();
+        private final SafeProc safeProc = new SafeProc();
+
+        public StringBased(StringValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public BytesRef getValue(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            scratch.copyChars(value);
+            return scratch;
+        }
+
+        @Override
+        public BytesRef getValueScratch(int docId, BytesRef ret) {
+            String value = values.getValue(docId);
+            if (value == null) {
+                ret.length = 0;
+                return ret;
+            }
+            ret.copyChars(value);
+            return ret;
+        }
+
+        @Override
+        public BytesRef getValueSafe(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            return new BytesRef(value);
+        }
+
+        @Override
+        public BytesRefArrayRef getValues(int docId) {
+            StringArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return BytesRefArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                String value = arrayRef.values[i];
+                arrayScratch.values[arrayScratch.end++] = value == null ? null : new BytesRef(value);
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public Iter getIterSafe(int docId) {
+            return safeValueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private final BytesRef scratch = new BytesRef();
+            private StringValues.Iter iter;
+
+            public ValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public BytesRef next() {
+                scratch.copyChars(iter.next());
+                return scratch;
+            }
+        }
+
+        static class SafeValueIter implements Iter {
+
+            private StringValues.Iter iter;
+
+            public SafeValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public BytesRef next() {
+                return new BytesRef(iter.next());
+            }
+        }
+
+        static class Proc implements StringValues.ValueInDocProc {
+
+            private final BytesRef scratch = new BytesRef();
+            private BytesValues.ValueInDocProc proc;
+
+            public Proc reset(BytesValues.ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                scratch.copyChars(value);
+                proc.onValue(docId, scratch);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+        static class SafeProc implements StringValues.ValueInDocProc {
+
+            private BytesValues.ValueInDocProc proc;
+
+            public SafeProc reset(BytesValues.ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                proc.onValue(docId, new BytesRef(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java b/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
new file mode 100644
index 00000000000..3e3d595cb61
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+
+/**
+ */
+public interface DoubleValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    double getValue(int docId);
+
+    double getValueMissing(int docId, double missingValue);
+
+    DoubleArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, double value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        double next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public double next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public double value;
+            public boolean done;
+
+            public Single reset(double value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public double next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java b/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
new file mode 100644
index 00000000000..f354a154c97
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.common.Nullable;
+
+/**
+ */
+public class FieldDataType {
+
+    private final String type;
+    @Nullable
+    private final String format;
+    private final ImmutableMap<String, String> options;
+
+    public FieldDataType(String type) {
+        this(type, null, ImmutableMap.<String, String>of());
+    }
+
+    public FieldDataType(String type, @Nullable String format, ImmutableMap<String, String> options) {
+        this.type = type;
+        this.format = format;
+        this.options = options;
+    }
+
+    public String getType() {
+        return this.type;
+    }
+
+    @Nullable
+    public String getFormat() {
+        return this.format;
+    }
+
+    public ImmutableMap<String, String> getOptions() {
+        return this.options;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/FloatValues.java b/src/main/java/org/elasticsearch/index/fielddata/FloatValues.java
new file mode 100644
index 00000000000..d03fdbd3341
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/FloatValues.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.FloatArrayRef;
+
+/**
+ */
+public interface FloatValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    float getValue(int docId);
+
+    float getValueMissing(int docId, float missingValue);
+
+    FloatArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, float value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        float next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public float next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public float value;
+            public boolean done;
+
+            public Single reset(float value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public float next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class DoubleBased implements FloatValues {
+
+        private final DoubleValues values;
+
+        private final FloatArrayRef arrayScratch = new FloatArrayRef(new float[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public DoubleBased(DoubleValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public float getValue(int docId) {
+            return (float) values.getValue(docId);
+        }
+
+        @Override
+        public float getValueMissing(int docId, float missingValue) {
+            return (float) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public FloatArrayRef getValues(int docId) {
+            DoubleArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return FloatArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (float) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private DoubleValues.Iter iter;
+
+            public ValueIter reset(DoubleValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public float next() {
+                return (float) iter.next();
+            }
+        }
+
+        static class Proc implements DoubleValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, double value) {
+                proc.onValue(docId, (float) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
new file mode 100644
index 00000000000..0658a4797a1
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+
+/**
+ */
+public interface HashedBytesValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    /**
+     * Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
+     */
+    HashedBytesRef getValue(int docId);
+
+    /**
+     * Returns a bytes value for a docId. The content is guaranteed not to be shared.
+     */
+    HashedBytesRef getValueSafe(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
+     */
+    Iter getIter(int docId);
+
+    /**
+     * Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
+     */
+    Iter getIterSafe(int docId);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    /**
+     * Go over all the possible values in their BytesRef format for a specific doc.
+     */
+    void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
+
+    public static interface ValueInDocProc {
+        void onValue(int docId, HashedBytesRef value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        HashedBytesRef next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public HashedBytesRef value;
+            public boolean done;
+
+            public Single reset(HashedBytesRef value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    /**
+     * A {@link BytesValues} based implementation.
+     */
+    static class BytesBased implements HashedBytesValues {
+
+        private final BytesValues values;
+
+        protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+        private final ValueIter valueIter = new ValueIter();
+        private final SafeValueIter safeValueIter = new SafeValueIter();
+        private final Proc proc = new Proc();
+        private final SafeProc safeProc = new SafeProc();
+
+        public BytesBased(BytesValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            BytesRef value = values.getValue(docId);
+            if (value == null) return null;
+            scratch.bytes = value;
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getValueSafe(int docId) {
+            return new HashedBytesRef(values.getValueSafe(docId));
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public Iter getIterSafe(int docId) {
+            return safeValueIter.reset(values.getIterSafe(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+            private BytesValues.Iter iter;
+
+            public ValueIter reset(BytesValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                scratch.bytes = iter.next();
+                return scratch.resetHashCode();
+            }
+        }
+
+        static class SafeValueIter implements Iter {
+
+            private BytesValues.Iter iter;
+
+            public SafeValueIter reset(BytesValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                return new HashedBytesRef(iter.next());
+            }
+        }
+
+        static class Proc implements BytesValues.ValueInDocProc {
+
+            private final HashedBytesRef scratch = new HashedBytesRef();
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, BytesRef value) {
+                scratch.bytes = value;
+                proc.onValue(docId, scratch.resetHashCode());
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+        static class SafeProc implements BytesValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public SafeProc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, BytesRef value) {
+                proc.onValue(docId, new HashedBytesRef(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+    }
+
+    static class StringBased implements HashedBytesValues {
+
+        private final StringValues values;
+
+        protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+        private final ValueIter valueIter = new ValueIter();
+        private final SafeValueIter safeValueIter = new SafeValueIter();
+        private final Proc proc = new Proc();
+        private final SafeProc safeProc = new SafeProc();
+
+        public StringBased(StringValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            scratch.bytes.copyChars(value);
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getValueSafe(int docId) {
+            String value = values.getValue(docId);
+            if (value == null) return null;
+            return new HashedBytesRef(new BytesRef(values.getValue(docId)));
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public Iter getIterSafe(int docId) {
+            return safeValueIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+            private StringValues.Iter iter;
+
+            public ValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                scratch.bytes.copyChars(iter.next());
+                return scratch.resetHashCode();
+            }
+        }
+
+        static class SafeValueIter implements Iter {
+
+            private StringValues.Iter iter;
+
+            public SafeValueIter reset(StringValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public HashedBytesRef next() {
+                return new HashedBytesRef(new BytesRef(iter.next()));
+            }
+        }
+
+        static class Proc implements StringValues.ValueInDocProc {
+
+            private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                scratch.bytes.copyChars(value);
+                proc.onValue(docId, scratch);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+
+        static class SafeProc implements StringValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public SafeProc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, String value) {
+                proc.onValue(docId, new HashedBytesRef(new BytesRef(value)));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
new file mode 100644
index 00000000000..1b1d1d34dd2
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexComponent;
+import org.elasticsearch.index.settings.IndexSettings;
+
+/**
+ */
+public interface IndexFieldData<FD extends AtomicFieldData> extends IndexComponent {
+
+    /**
+     * The field name.
+     */
+    String getFieldName();
+
+    /**
+     * Are the values ordered? (in ascending manner).
+     */
+    boolean valuesOrdered();
+
+    /**
+     * Loads the atomic field data for the reader, possibly cached.
+     */
+    FD load(AtomicReaderContext context);
+
+    /**
+     * Loads directly the atomic field data for the reader, ignoring any caching involved.
+     */
+    FD loadDirect(AtomicReaderContext context) throws Exception;
+
+    /**
+     * Comparator used for sorting.
+     */
+    XFieldComparatorSource comparatorSource(@Nullable Object missingValue);
+
+    /**
+     * Clears any resources associated with this field data.
+     */
+    void clear();
+
+    // we need this extended source we we have custom comparators to reuse our field data
+    // in this case, we need to reduce type that will be used when search results are reduced
+    // on another node (we don't have the custom source them...)
+    public abstract class XFieldComparatorSource extends FieldComparatorSource {
+
+        public abstract SortField.Type reducedType();
+    }
+
+    interface Builder {
+
+        IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
new file mode 100644
index 00000000000..3feb5a19f87
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.SegmentReader;
+import org.elasticsearch.index.Index;
+
+import java.util.concurrent.Callable;
+
+/**
+ * A simple field data cache abstraction.
+ */
+public interface IndexFieldDataCache {
+
+    <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(AtomicReaderContext context, IFD indexFieldData) throws Exception;
+
+    void clear(Index index);
+
+    void clear(Index index, String fieldName);
+
+    /**
+     * The resident field data cache is a *per field* cache that keeps all the values in memory.
+     */
+    static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener {
+        private final Cache<Object, AtomicFieldData> cache;
+
+        protected FieldBased(Cache<Object, AtomicFieldData> cache) {
+            this.cache = cache;
+        }
+
+        @Override
+        public void onClose(SegmentReader owner) {
+            cache.invalidate(owner.getCoreCacheKey());
+        }
+
+        @Override
+        public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
+            //noinspection unchecked
+            return (FD) cache.get(context.reader().getCoreCacheKey(), new Callable<AtomicFieldData>() {
+                @Override
+                public AtomicFieldData call() throws Exception {
+                    return indexFieldData.loadDirect(context);
+                }
+            });
+        }
+
+        @Override
+        public void clear(Index index) {
+            cache.invalidateAll();
+        }
+
+        @Override
+        public void clear(Index index, String fieldName) {
+            cache.invalidateAll();
+        }
+    }
+
+    static class Resident extends FieldBased {
+
+        public Resident() {
+            super(CacheBuilder.newBuilder().<Object, AtomicFieldData>build());
+        }
+    }
+
+    static class Soft extends FieldBased {
+
+        public Soft() {
+            super(CacheBuilder.newBuilder().softValues().<Object, AtomicFieldData>build());
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
new file mode 100644
index 00000000000..a536dd3af1e
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.plain.ConcreteBytesRefIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ */
+public class IndexFieldDataService extends AbstractIndexComponent {
+
+    private final static ImmutableMap<String, IndexFieldData.Builder> buildersByType;
+    private final static ImmutableMap<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
+
+    static {
+        buildersByType = MapBuilder.<String, IndexFieldData.Builder>newMapBuilder()
+                .put("string", new ConcreteBytesRefIndexFieldData.Builder())
+                .put("double", new DoubleArrayIndexFieldData.Builder())
+                .immutableMap();
+
+        buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder().
+                put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
+                .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
+                .immutableMap();
+    }
+
+    private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
+
+    public IndexFieldDataService(Index index) {
+        this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
+    }
+
+    @Inject
+    public IndexFieldDataService(Index index, @IndexSettings Settings indexSettings) {
+        super(index, indexSettings);
+    }
+
+    public void clear() {
+        synchronized (loadedFieldData) {
+            for (IndexFieldData fieldData : loadedFieldData.values()) {
+                fieldData.clear();
+            }
+            loadedFieldData.clear();
+        }
+    }
+
+    public void clearField(String fieldName) {
+        synchronized (loadedFieldData) {
+            IndexFieldData fieldData = loadedFieldData.remove(fieldName);
+            if (fieldData != null) {
+                fieldData.clear();
+            }
+        }
+    }
+
+    public <IFD extends IndexFieldData> IFD getForField(String fieldName, FieldDataType type) {
+        IndexFieldData fieldData = loadedFieldData.get(type.getType());
+        if (fieldData == null) {
+            synchronized (loadedFieldData) {
+                fieldData = loadedFieldData.get(type.getType());
+                if (fieldData == null) {
+                    IndexFieldData.Builder builder = null;
+                    if (type.getFormat() != null) {
+                        builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), type.getFormat()));
+                    }
+                    if (builder == null) {
+                        builder = buildersByType.get(type.getType());
+                    }
+                    if (builder == null) {
+                        throw new ElasticSearchIllegalArgumentException("failed to find field data builder for field " + fieldName + ", and type " + type);
+                    }
+
+                    IndexFieldDataCache cache;
+                    if (type.getOptions().containsKey("cache")) {
+                        String cacheType = type.getOptions().get("cache");
+                        if ("resident".equals(cacheType)) {
+                            cache = new IndexFieldDataCache.Resident();
+                        } else if ("soft".equals(cacheType)) {
+                            cache = new IndexFieldDataCache.Soft();
+                        } else {
+                            throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]");
+                        }
+                    } else {
+                        cache = new IndexFieldDataCache.Resident();
+                    }
+
+                    fieldData = builder.build(index, indexSettings, fieldName, type, cache);
+                    loadedFieldData.put(fieldName, fieldData);
+                }
+            }
+        }
+        return (IFD) fieldData;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
new file mode 100644
index 00000000000..90633cba92a
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.AtomicReaderContext;
+
+/**
+ */
+public interface IndexNumericFieldData<FD extends AtomicNumericFieldData> extends IndexFieldData<FD> {
+
+    static enum NumericType {
+        BYTE,
+        SHORT,
+        INT,
+        LONG,
+        FLOAT,
+        DOUBLE
+    }
+
+    NumericType getNumericType();
+
+    /**
+     * Loads the atomic field data for the reader, possibly cached.
+     */
+    FD load(AtomicReaderContext context);
+
+    /**
+     * Loads directly the atomic field data for the reader, ignoring any caching involved.
+     */
+    FD loadDirect(AtomicReaderContext context) throws Exception;
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalFieldData.java
new file mode 100644
index 00000000000..11d08c280de
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalFieldData.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.AtomicReaderContext;
+
+/**
+ */
+public interface IndexOrdinalFieldData<FD extends AtomicOrdinalFieldData> extends IndexFieldData<FD> {
+
+    /**
+     * Loads the atomic field data for the reader, possibly cached.
+     */
+    FD load(AtomicReaderContext context);
+
+    /**
+     * Loads directly the atomic field data for the reader, ignoring any caching involved.
+     */
+    FD loadDirect(AtomicReaderContext context) throws Exception;
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IntValues.java b/src/main/java/org/elasticsearch/index/fielddata/IntValues.java
new file mode 100644
index 00000000000..1a9f5908dee
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IntValues.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public interface IntValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    int getValue(int docId);
+
+    int getValueMissing(int docId, int missingValue);
+
+    IntArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, int value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        int next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public int next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public int value;
+            public boolean done;
+
+            public Single reset(int value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public int next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class LongBased implements IntValues {
+
+        private final LongValues values;
+
+        private final IntArrayRef arrayScratch = new IntArrayRef(new int[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public int getValue(int docId) {
+            return (int) values.getValue(docId);
+        }
+
+        @Override
+        public int getValueMissing(int docId, int missingValue) {
+            return (int) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public IntArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return IntArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (int) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            public ValueIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public int next() {
+                return (int) iter.next();
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                proc.onValue(docId, (int) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/LongValues.java b/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
new file mode 100644
index 00000000000..f6de0d87233
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public interface LongValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    long getValue(int docId);
+
+    long getValueMissing(int docId, long missingValue);
+
+    LongArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, long value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        long next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public long next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public long value;
+            public boolean done;
+
+            public Single reset(long value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public long next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java
new file mode 100644
index 00000000000..a51fa893afa
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+/**
+ */
+public interface OrdinalsBytesValues extends BytesValues {
+
+    Ordinals.Docs ordinals();
+
+    BytesRef getValueByOrd(int ord);
+
+    /**
+     * Returns the bytes value for the docId, with the provided "ret" which will be filled with the
+     * result which will also be returned. If there is no value for this docId, the length will be 0.
+     * Note, the bytes are not "safe".
+     */
+    BytesRef getValueScratchByOrd(int ord, BytesRef ret);
+
+    BytesRef getSafeValueByOrd(int ord);
+
+    public static class StringBased extends BytesValues.StringBased implements OrdinalsBytesValues {
+
+        private final OrdinalsStringValues values;
+
+        public StringBased(OrdinalsStringValues values) {
+            super(values);
+            this.values = values;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return values.ordinals();
+        }
+
+        @Override
+        public BytesRef getValueByOrd(int ord) {
+            scratch.copyChars(values.getValueByOrd(ord));
+            return scratch;
+        }
+
+        @Override
+        public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
+            ret.copyChars(values.getValueByOrd(ord));
+            return ret;
+        }
+
+        @Override
+        public BytesRef getSafeValueByOrd(int ord) {
+            return new BytesRef(values.getValueByOrd(ord));
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java
new file mode 100644
index 00000000000..6f109a9a233
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+/**
+ */
+public interface OrdinalsHashedBytesValues extends HashedBytesValues {
+
+    Ordinals.Docs ordinals();
+
+    HashedBytesRef getValueByOrd(int ord);
+
+    HashedBytesRef getSafeValueByOrd(int ord);
+
+    static class BytesBased extends HashedBytesValues.BytesBased implements OrdinalsHashedBytesValues {
+
+        private final OrdinalsBytesValues values;
+
+        public BytesBased(OrdinalsBytesValues values) {
+            super(values);
+            this.values = values;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return values.ordinals();
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            scratch.bytes = values.getValueByOrd(ord);
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return new HashedBytesRef(values.getSafeValueByOrd(ord));
+        }
+    }
+
+    static class StringBased extends HashedBytesValues.StringBased implements OrdinalsHashedBytesValues {
+
+        private final OrdinalsStringValues values;
+
+        public StringBased(OrdinalsStringValues values) {
+            super(values);
+            this.values = values;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return values.ordinals();
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            scratch.bytes.copyChars(values.getValueByOrd(ord));
+            return scratch.resetHashCode();
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return new HashedBytesRef(new BytesRef(values.getValueByOrd(ord)));
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java
new file mode 100644
index 00000000000..9d8c0e1d44a
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+/**
+ */
+public interface OrdinalsStringValues extends StringValues {
+
+    Ordinals.Docs ordinals();
+
+    String getValueByOrd(int ord);
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
new file mode 100644
index 00000000000..bf60565b769
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.util.*;
+
+/**
+ * Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
+ * and a <code>getValues</code> that return the relevant type that then can be used in scripts.
+ */
+public interface ScriptDocValues {
+
+    void setNextDocId(int docId);
+
+    boolean isEmpty();
+
+    static class Strings implements ScriptDocValues {
+
+        private final StringValues values;
+        private int docId;
+
+        public Strings(StringValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public String getValue() {
+            return values.getValue(docId);
+        }
+
+        public StringArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class Bytes implements ScriptDocValues {
+
+        private final BytesValues values;
+        private int docId;
+
+        public Bytes(BytesValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public BytesRef getValue() {
+            return values.getValue(docId);
+        }
+
+        public BytesRefArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericByte implements ScriptDocValues {
+
+        private final ByteValues values;
+        private int docId;
+
+        public NumericByte(ByteValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public byte getValue() {
+            return values.getValue(docId);
+        }
+
+        public ByteArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericShort implements ScriptDocValues {
+
+        private final ShortValues values;
+        private int docId;
+
+        public NumericShort(ShortValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public short getValue() {
+            return values.getValue(docId);
+        }
+
+        public ShortArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericInteger implements ScriptDocValues {
+
+        private final IntValues values;
+        private int docId;
+
+        public NumericInteger(IntValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public int getValue() {
+            return values.getValue(docId);
+        }
+
+        public IntArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericLong implements ScriptDocValues {
+
+        private final LongValues values;
+        private int docId;
+
+        public NumericLong(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public long getValue() {
+            return values.getValue(docId);
+        }
+
+        public LongArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericFloat implements ScriptDocValues {
+
+        private final FloatValues values;
+        private int docId;
+
+        public NumericFloat(FloatValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public float getValue() {
+            return values.getValue(docId);
+        }
+
+        public FloatArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+
+    static class NumericDouble implements ScriptDocValues {
+
+        private final DoubleValues values;
+        private int docId;
+
+        public NumericDouble(DoubleValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public double getValue() {
+            return values.getValue(docId);
+        }
+
+        public DoubleArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java b/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
new file mode 100644
index 00000000000..43de95e1369
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+import org.elasticsearch.index.fielddata.util.ShortArrayRef;
+
+/**
+ */
+public interface ShortValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    short getValue(int docId);
+
+    short getValueMissing(int docId, short missingValue);
+
+    ShortArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    static interface ValueInDocProc {
+        void onValue(int docId, short value);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        short next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public short next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public short value;
+            public boolean done;
+
+            public Single reset(short value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public short next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+
+    public static class LongBased implements ShortValues {
+
+        private final LongValues values;
+
+        private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public short getValue(int docId) {
+            return (short) values.getValue(docId);
+        }
+
+        @Override
+        public short getValueMissing(int docId, short missingValue) {
+            return (short) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public ShortArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return ShortArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (short) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            public ValueIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public short next() {
+                return (short) iter.next();
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                proc.onValue(docId, (short) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
new file mode 100644
index 00000000000..d017568f328
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public interface StringValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    String getValue(int docId);
+
+    StringArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    /**
+     * Go over all the possible values.
+     */
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    public static interface ValueInDocProc {
+        void onValue(int docId, String value);
+
+        void onMissing(int docId);
+    }
+
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        String next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public String next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public String value;
+            public boolean done;
+
+            public Single reset(String value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public String next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
new file mode 100644
index 00000000000..e40a090f753
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexFieldData indexFieldData;
+
+    public BytesRefFieldComparatorSource(IndexFieldData indexFieldData) {
+        this.indexFieldData = indexFieldData;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.STRING;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldName());
+        if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexOrdinalFieldData) {
+            return new BytesRefOrdValComparator((IndexOrdinalFieldData) indexFieldData, numHits);
+        }
+        return new BytesRefValComparator(indexFieldData, numHits);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
new file mode 100644
index 00000000000..0bbfbdb2835
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
@@ -0,0 +1,475 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
+import org.elasticsearch.index.fielddata.OrdinalsBytesValues;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+
+import java.io.IOException;
+
+/**
+ * Sorts by field's natural Term sort order, using
+ * ordinals.  This is functionally equivalent to {@link
+ * org.apache.lucene.search.FieldComparator.TermValComparator}, but it first resolves the string
+ * to their relative ordinal positions (using the index
+ * returned by {@link org.apache.lucene.search.FieldCache#getTermsIndex}), and
+ * does most comparisons using the ordinals.  For medium
+ * to large results, this comparator will be much faster
+ * than {@link org.apache.lucene.search.FieldComparator.TermValComparator}.  For very small
+ * result sets it may be slower.
+ */
+public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
+
+    final IndexOrdinalFieldData indexFieldData;
+
+    /* Ords for each slot.
+       @lucene.internal */
+    final int[] ords;
+
+    /* Values for each slot.
+       @lucene.internal */
+    final BytesRef[] values;
+
+    /* Which reader last copied a value into the slot. When
+       we compare two slots, we just compare-by-ord if the
+       readerGen is the same; else we must compare the
+       values (slower).
+       @lucene.internal */
+    final int[] readerGen;
+
+    /* Gen of current reader we are on.
+       @lucene.internal */
+    int currentReaderGen = -1;
+
+    /* Current reader's doc ord/values.
+       @lucene.internal */
+    OrdinalsBytesValues termsIndex;
+
+    /* Bottom slot, or -1 if queue isn't full yet
+       @lucene.internal */
+    int bottomSlot = -1;
+
+    /* Bottom ord (same as ords[bottomSlot] once bottomSlot
+       is set).  Cached for faster compares.
+       @lucene.internal */
+    int bottomOrd;
+
+    /* True if current bottom slot matches the current
+       reader.
+       @lucene.internal */
+    boolean bottomSameReader;
+
+    /* Bottom value (same as values[bottomSlot] once
+       bottomSlot is set).  Cached for faster compares.
+      @lucene.internal */
+    BytesRef bottomValue;
+
+    final BytesRef tempBR = new BytesRef();
+
+    public BytesRefOrdValComparator(IndexOrdinalFieldData indexFieldData, int numHits) {
+        this.indexFieldData = indexFieldData;
+        ords = new int[numHits];
+        values = new BytesRef[numHits];
+        readerGen = new int[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        if (readerGen[slot1] == readerGen[slot2]) {
+            return ords[slot1] - ords[slot2];
+        }
+
+        final BytesRef val1 = values[slot1];
+        final BytesRef val2 = values[slot2];
+        if (val1 == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+        return val1.compareTo(val2);
+    }
+
+    @Override
+    public int compareBottom(int doc) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void copy(int slot, int doc) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int compareDocToValue(int doc, BytesRef value) {
+        BytesRef docValue = termsIndex.getValue(doc);
+        if (docValue == null) {
+            if (value == null) {
+                return 0;
+            }
+            return -1;
+        } else if (value == null) {
+            return 1;
+        }
+        return docValue.compareTo(value);
+    }
+
+    /**
+     * Base class for specialized (per bit width of the
+     * ords) per-segment comparator.  NOTE: this is messy;
+     * we do this only because hotspot can't reliably inline
+     * the underlying array access when looking up doc->ord
+     *
+     * @lucene.internal
+     */
+    abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
+
+        @Override
+        public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+            return BytesRefOrdValComparator.this.setNextReader(context);
+        }
+
+        @Override
+        public int compare(int slot1, int slot2) {
+            return BytesRefOrdValComparator.this.compare(slot1, slot2);
+        }
+
+        @Override
+        public void setBottom(final int bottom) {
+            BytesRefOrdValComparator.this.setBottom(bottom);
+        }
+
+        @Override
+        public BytesRef value(int slot) {
+            return BytesRefOrdValComparator.this.value(slot);
+        }
+
+        @Override
+        public int compareValues(BytesRef val1, BytesRef val2) {
+            if (val1 == null) {
+                if (val2 == null) {
+                    return 0;
+                }
+                return -1;
+            } else if (val2 == null) {
+                return 1;
+            }
+            return val1.compareTo(val2);
+        }
+
+        @Override
+        public int compareDocToValue(int doc, BytesRef value) {
+            return BytesRefOrdValComparator.this.compareDocToValue(doc, value);
+        }
+    }
+
+    // Used per-segment when bit width of doc->ord is 8:
+    private final class ByteOrdComparator extends PerSegmentComparator {
+        private final byte[] readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public ByteOrdComparator(byte[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
+            this.readerOrds = readerOrds;
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = (readerOrds[doc] & 0xFF);
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds[doc] & 0xFF;
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    // Used per-segment when bit width of doc->ord is 16:
+    private final class ShortOrdComparator extends PerSegmentComparator {
+        private final short[] readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public ShortOrdComparator(short[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
+            this.readerOrds = readerOrds;
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = (readerOrds[doc] & 0xFFFF);
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds[doc] & 0xFFFF;
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    // Used per-segment when bit width of doc->ord is 32:
+    private final class IntOrdComparator extends PerSegmentComparator {
+        private final int[] readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public IntOrdComparator(int[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
+            this.readerOrds = readerOrds;
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = readerOrds[doc];
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds[doc];
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    // Used per-segment when bit width is not a native array
+    // size (8, 16, 32):
+    final class AnyOrdComparator extends PerSegmentComparator {
+        private final IndexFieldData fieldData;
+        private final Ordinals.Docs readerOrds;
+        private final OrdinalsBytesValues termsIndex;
+        private final int docBase;
+
+        public AnyOrdComparator(IndexFieldData fieldData, OrdinalsBytesValues termsIndex, int docBase) {
+            this.fieldData = fieldData;
+            this.readerOrds = termsIndex.ordinals();
+            this.termsIndex = termsIndex;
+            this.docBase = docBase;
+        }
+
+        @Override
+        public int compareBottom(int doc) {
+            assert bottomSlot != -1;
+            final int docOrd = readerOrds.getOrd(doc);
+            if (bottomSameReader) {
+                // ord is precisely comparable, even in the equal case
+                return bottomOrd - docOrd;
+            } else if (bottomOrd >= docOrd) {
+                // the equals case always means bottom is > doc
+                // (because we set bottomOrd to the lower bound in
+                // setBottom):
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+
+        @Override
+        public void copy(int slot, int doc) {
+            final int ord = readerOrds.getOrd(doc);
+            ords[slot] = ord;
+            if (ord == 0) {
+                values[slot] = null;
+            } else {
+                assert ord > 0;
+                if (values[slot] == null) {
+                    values[slot] = new BytesRef();
+                }
+                termsIndex.getValueScratchByOrd(ord, values[slot]);
+            }
+            readerGen[slot] = currentReaderGen;
+        }
+    }
+
+    @Override
+    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+        final int docBase = context.docBase;
+        termsIndex = indexFieldData.load(context).getBytesValues();
+        // TODO, we should support sorting on multi valued field, take the best ascending value out of all the values
+        if (termsIndex.isMultiValued()) {
+            throw new ElasticSearchIllegalArgumentException("can't sort on a multi valued field");
+        }
+        final Ordinals.Docs docToOrd = termsIndex.ordinals();
+        Object ordsStorage = docToOrd.ordinals().getBackingStorage();
+        FieldComparator<BytesRef> perSegComp = null;
+
+        if (ordsStorage instanceof byte[]) {
+            perSegComp = new ByteOrdComparator((byte[]) ordsStorage, termsIndex, docBase);
+        } else if (ordsStorage instanceof short[]) {
+            perSegComp = new ShortOrdComparator((short[]) ordsStorage, termsIndex, docBase);
+        } else if (ordsStorage instanceof int[]) {
+            perSegComp = new IntOrdComparator((int[]) ordsStorage, termsIndex, docBase);
+        }
+        // Don't specialize the long[] case since it's not
+        // possible, ie, worse case is MAX_INT-1 docs with
+        // every one having a unique value.
+
+        // TODO: ES - should we optimize for the PackedInts.Reader case as well?
+        if (perSegComp == null) {
+            perSegComp = new AnyOrdComparator(indexFieldData, termsIndex, docBase);
+        }
+
+        currentReaderGen++;
+        if (bottomSlot != -1) {
+            perSegComp.setBottom(bottomSlot);
+        }
+
+        return perSegComp;
+    }
+
+    @Override
+    public void setBottom(final int bottom) {
+        bottomSlot = bottom;
+
+        bottomValue = values[bottomSlot];
+        if (currentReaderGen == readerGen[bottomSlot]) {
+            bottomOrd = ords[bottomSlot];
+            bottomSameReader = true;
+        } else {
+            if (bottomValue == null) {
+                // 0 ord is null for all segments
+                assert ords[bottomSlot] == 0;
+                bottomOrd = 0;
+                bottomSameReader = true;
+                readerGen[bottomSlot] = currentReaderGen;
+            } else {
+                final int index = binarySearch(termsIndex, bottomValue);
+                if (index < 0) {
+                    bottomOrd = -index - 2;
+                    bottomSameReader = false;
+                } else {
+                    bottomOrd = index;
+                    // exact value match
+                    bottomSameReader = true;
+                    readerGen[bottomSlot] = currentReaderGen;
+                    ords[bottomSlot] = bottomOrd;
+                }
+            }
+        }
+    }
+
+    @Override
+    public BytesRef value(int slot) {
+        return values[slot];
+    }
+
+    final protected static int binarySearch(OrdinalsBytesValues a, BytesRef key) {
+        return binarySearch(a, key, 1, a.ordinals().getNumDocs() - 1);
+    }
+
+    final protected static int binarySearch(OrdinalsBytesValues a, BytesRef key, int low, int high) {
+
+        while (low <= high) {
+            int mid = (low + high) >>> 1;
+            BytesRef midVal = a.getValueByOrd(mid);
+            int cmp;
+            if (midVal != null) {
+                cmp = midVal.compareTo(key);
+            } else {
+                cmp = -1;
+            }
+
+            if (cmp < 0)
+                low = mid + 1;
+            else if (cmp > 0)
+                high = mid - 1;
+            else
+                return mid;
+        }
+        return -(low + 1);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
new file mode 100644
index 00000000000..6544fd99dcf
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.BytesValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+
+import java.io.IOException;
+
+/**
+ * Sorts by field's natural Term sort order.  All
+ * comparisons are done using BytesRef.compareTo, which is
+ * slow for medium to large result sets but possibly
+ * very fast for very small results sets.
+ */
+public final class BytesRefValComparator extends FieldComparator<BytesRef> {
+
+    private final IndexFieldData indexFieldData;
+    private BytesRef[] values;
+    private BytesValues docTerms;
+    private BytesRef bottom;
+
+    BytesRefValComparator(IndexFieldData indexFieldData, int numHits) {
+        values = new BytesRef[numHits];
+        this.indexFieldData = indexFieldData;
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final BytesRef val1 = values[slot1];
+        final BytesRef val2 = values[slot2];
+        if (val1 == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+
+        return val1.compareTo(val2);
+    }
+
+    @Override
+    public int compareBottom(int doc) {
+        BytesRef val2 = docTerms.getValue(doc);
+        if (bottom == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+        return bottom.compareTo(val2);
+    }
+
+    @Override
+    public void copy(int slot, int doc) {
+        if (values[slot] == null) {
+            values[slot] = new BytesRef();
+        }
+        docTerms.getValueScratch(doc, values[slot]);
+    }
+
+    @Override
+    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+        docTerms = indexFieldData.load(context).getBytesValues();
+        return this;
+    }
+
+    @Override
+    public void setBottom(final int bottom) {
+        this.bottom = values[bottom];
+    }
+
+    @Override
+    public BytesRef value(int slot) {
+        return values[slot];
+    }
+
+    @Override
+    public int compareValues(BytesRef val1, BytesRef val2) {
+        if (val1 == null) {
+            if (val2 == null) {
+                return 0;
+            }
+            return -1;
+        } else if (val2 == null) {
+            return 1;
+        }
+        return val1.compareTo(val2);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, BytesRef value) {
+        return docTerms.getValue(doc).compareTo(value);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java
new file mode 100644
index 00000000000..5d78cff68ef
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class DoubleValuesComparator extends FieldComparator<Double> {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final double missingValue;
+
+    protected final double[] values;
+    private double bottom;
+    private DoubleValues readerValues;
+
+    public DoubleValuesComparator(IndexNumericFieldData indexFieldData, double missingValue, int numHits) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+        this.values = new double[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final double v1 = values[slot1];
+        final double v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void setBottom(int slot) {
+        this.bottom = values[slot];
+    }
+
+    @Override
+    public int compareBottom(int doc) throws IOException {
+        double v2 = readerValues.getValueMissing(doc, missingValue);
+
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void copy(int slot, int doc) throws IOException {
+        values[slot] = readerValues.getValueMissing(doc, missingValue);
+    }
+
+    @Override
+    public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getDoubleValues();
+        return this;
+    }
+
+    @Override
+    public Double value(int slot) {
+        return Double.valueOf(values[slot]);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Double valueObj) throws IOException {
+        final double value = valueObj.doubleValue();
+        double docValue = readerValues.getValueMissing(doc, missingValue);
+        if (docValue < value) {
+            return -1;
+        } else if (docValue > value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
new file mode 100644
index 00000000000..84f001f631f
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final Object missingValue;
+
+    public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.DOUBLE;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldName());
+
+        double dMissingValue;
+        if (missingValue == null || "_last".equals(missingValue)) {
+            dMissingValue = reversed ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
+        } else if ("_first".equals(missingValue)) {
+            dMissingValue = reversed ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
+        } else {
+            dMissingValue = missingValue instanceof Number ? ((Number) missingValue).doubleValue() : Double.parseDouble(missingValue.toString());
+        }
+
+        return new DoubleValuesComparator(indexFieldData, dMissingValue, numHits);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java
new file mode 100644
index 00000000000..dfc9e68dcd6
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ * Ordinals that effectively are single valued and map "one to one" to the
+ * doc ids. Note, the docId is incremented by 1 to get the ordinal, since 0
+ * denotes an empty value.
+ */
+public class DocIdOrdinals implements Ordinals {
+
+    private final int numDocs;
+
+    /**
+     * Constructs a new doc id ordinals.
+     */
+    public DocIdOrdinals(int numDocs) {
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return null;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        return RamUsage.NUM_BYTES_OBJECT_REF;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numDocs + 1;
+    }
+
+    @Override
+    public Ordinals.Docs ordinals() {
+        return new Docs(this);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final DocIdOrdinals parent;
+        private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
+        private final SingleValueIter iter = new SingleValueIter();
+
+        public Docs(DocIdOrdinals parent) {
+            this.parent = parent;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return docId + 1;
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            intsScratch.values[0] = docId + 1;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(docId + 1);
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, docId + 1);
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java
new file mode 100644
index 00000000000..b9d6b9c7e14
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ */
+public class EmptyOrdinals implements Ordinals {
+
+    private final int numDocs;
+
+    public EmptyOrdinals(int numDocs) {
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        return 0;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return null;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return this.numDocs;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numDocs + 1;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final EmptyOrdinals parent;
+
+        public Docs(EmptyOrdinals parent) {
+            this.parent = parent;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return 0;
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            return IntArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return EmptyIter.INSTANCE;
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, 0);
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiFlatArrayOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiFlatArrayOrdinals.java
new file mode 100644
index 00000000000..11a2dac53f8
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiFlatArrayOrdinals.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ * "Flat" multi valued ordinals, the first level array size is as the maximum
+ * values a docId has. Ordinals are populated in order from the first flat array
+ * value to the next.
+ */
+public class MultiFlatArrayOrdinals implements Ordinals {
+
+    private ThreadLocal<IntArrayRef> intArrayRefCache = new ThreadLocal<IntArrayRef>() {
+        @Override
+        protected IntArrayRef initialValue() {
+            return new IntArrayRef(new int[ordinals.length]);
+        }
+    };
+
+    // ordinals with value 0 indicates no value
+    private final int[][] ordinals;
+    private final int numDocs;
+    private final int numOrds;
+
+    private long size = -1;
+
+    public MultiFlatArrayOrdinals(int[][] ordinals, int numOrds) {
+        assert ordinals.length > 0;
+        this.ordinals = ordinals;
+        this.numDocs = ordinals[0].length;
+        this.numOrds = numOrds;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return ordinals;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            long size = 0;
+            size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
+            for (int[] ordinal : ordinals) {
+                size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
+            }
+            this.size = size;
+        }
+        return size;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return true;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numOrds;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this, ordinals, intArrayRefCache.get());
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final MultiFlatArrayOrdinals parent;
+        private final int[][] ordinals;
+        private final IterImpl iter;
+
+        private final IntArrayRef intsScratch;
+
+        public Docs(MultiFlatArrayOrdinals parent, int[][] ordinals, IntArrayRef intsScratch) {
+            this.parent = parent;
+            this.ordinals = ordinals;
+            this.iter = new IterImpl(ordinals);
+            this.intsScratch = intsScratch;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return this.parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return true;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return ordinals[0][docId];
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            intsScratch.end = 0;
+            int i;
+            for (i = 0; i < ordinals.length; i++) {
+                int ordinal = ordinals[i][docId];
+                if (ordinal == 0) {
+                    if (i == 0) return IntArrayRef.EMPTY;
+                    break;
+                }
+                intsScratch.values[i] = ordinal;
+            }
+            intsScratch.end = i;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(docId);
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            for (int i = 0; i < ordinals.length; i++) {
+                int ordinal = ordinals[i][docId];
+                if (ordinal == 0) {
+                    if (i == 0) proc.onOrdinal(docId, 0);
+                    return;
+                }
+                proc.onOrdinal(docId, ordinal);
+            }
+        }
+
+        public static class IterImpl implements Docs.Iter {
+
+            private final int[][] ordinals;
+            private int docId;
+            private int i;
+
+            public IterImpl(int[][] ordinals) {
+                this.ordinals = ordinals;
+            }
+
+            public IterImpl reset(int docId) {
+                this.docId = docId;
+                this.i = 0;
+                return this;
+            }
+
+            @Override
+            public int next() {
+                if (i >= ordinals.length) return 0;
+                return ordinals[i++][docId];
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
new file mode 100644
index 00000000000..b2d25789db7
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ * A thread safe ordinals abstraction. Ordinals can only be positive integers.
+ */
+public interface Ordinals {
+
+    /**
+     * Returns the backing storage for this ordinals.
+     */
+    Object getBackingStorage();
+
+    /**
+     * The memory size this ordinals take.
+     */
+    long getMemorySizeInBytes();
+
+    /**
+     * Is one of the docs maps to more than one ordinal?
+     */
+    boolean isMultiValued();
+
+    /**
+     * The number of docs in this ordinals.
+     */
+    int getNumDocs();
+
+    /**
+     * The number of ordinals.
+     */
+    int getNumOrds();
+
+    /**
+     * Returns a lightweight (non thread safe) view iterator of the ordinals.
+     */
+    Docs ordinals();
+
+    /**
+     * A non thread safe ordinals abstraction, yet very lightweight to create. The idea
+     * is that this gets created for each "iteration" over ordinals.
+     * <p/>
+     * <p>A value of 0 ordinal when iterating indicated "no" value.</p>
+     */
+    interface Docs {
+
+        /**
+         * Returns the original ordinals used to generate this Docs "itereator".
+         */
+        Ordinals ordinals();
+
+        /**
+         * The number of docs in this ordinals.
+         */
+        int getNumDocs();
+
+        /**
+         * The number of ordinals.
+         */
+        int getNumOrds();
+
+        /**
+         * Is one of the docs maps to more than one ordinal?
+         */
+        boolean isMultiValued();
+
+        /**
+         * The ordinal that maps to the relevant docId. If it has no value, returns
+         * <tt>0</tt>.
+         */
+        int getOrd(int docId);
+
+        /**
+         * Returns an array of ordinals matching the docIds, with 0 length one
+         * for a doc with no ordinals.
+         */
+        IntArrayRef getOrds(int docId);
+
+        /**
+         * Returns an iterator of the ordinals that match the docId, with an
+         * empty iterator for a doc with no ordinals.
+         */
+        Iter getIter(int docId);
+
+        /**
+         * Iterates over the ordinals associated with a docId. If there are no values,
+         * a callback with a value 0 will be done.
+         */
+        void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc);
+
+        public static interface OrdinalInDocProc {
+            void onOrdinal(int docId, int ordinal);
+        }
+
+        /**
+         * An iterator over ordinals values.
+         */
+        interface Iter {
+
+            /**
+             * Gets the next ordinal. Returning 0 if the iteration is exhausted.
+             */
+            int next();
+        }
+
+        static class EmptyIter implements Iter {
+
+            public static EmptyIter INSTANCE = new EmptyIter();
+
+            @Override
+            public int next() {
+                return 0;
+            }
+        }
+
+        static class SingleValueIter implements Iter {
+
+            private int value;
+
+            public SingleValueIter reset(int value) {
+                this.value = value;
+                return this;
+            }
+
+            @Override
+            public int next() {
+                int actual = value;
+                value = 0;
+                return actual;
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java
new file mode 100644
index 00000000000..ae93b8ac2c0
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ */
+public class SingleArrayOrdinals implements Ordinals {
+
+    // ordinals with value 0 indicates no value
+    private final int[] ordinals;
+    private final int numOrds;
+
+    private long size = -1;
+
+    public SingleArrayOrdinals(int[] ordinals, int numOrds) {
+        this.ordinals = ordinals;
+        this.numOrds = numOrds;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        return ordinals;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            size = RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
+        }
+        return size;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return ordinals.length;
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numOrds;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this, ordinals);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final SingleArrayOrdinals parent;
+        private final int[] ordinals;
+
+        private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
+        private final SingleValueIter iter = new SingleValueIter();
+
+        public Docs(SingleArrayOrdinals parent, int[] ordinals) {
+            this.parent = parent;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return ordinals[docId];
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            int ordinal = ordinals[docId];
+            if (ordinal == 0) return IntArrayRef.EMPTY;
+            intsScratch.values[0] = docId;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(ordinals[docId]);
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, ordinals[docId]);
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
new file mode 100644
index 00000000000..5a3d69253aa
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.ordinals;
+
+import org.apache.lucene.util.packed.PackedInts;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+
+/**
+ */
+public class SinglePackedOrdinals implements Ordinals {
+
+    // ordinals with value 0 indicates no value
+    private final PackedInts.Reader reader;
+    private final int numOrds;
+
+    private long size = -1;
+
+    public SinglePackedOrdinals(PackedInts.Reader reader, int numOrds) {
+        this.reader = reader;
+        this.numOrds = numOrds;
+    }
+
+    @Override
+    public Object getBackingStorage() {
+        if (reader.hasArray()) {
+            return reader.getArray();
+        }
+        return reader;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            size = RamUsage.NUM_BYTES_OBJECT_REF + reader.ramBytesUsed();
+        }
+        return size;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return false;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return reader.size();
+    }
+
+    @Override
+    public int getNumOrds() {
+        return numOrds;
+    }
+
+    @Override
+    public Docs ordinals() {
+        return new Docs(this, reader);
+    }
+
+    public static class Docs implements Ordinals.Docs {
+
+        private final SinglePackedOrdinals parent;
+        private final PackedInts.Reader reader;
+
+        private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
+        private final SingleValueIter iter = new SingleValueIter();
+
+        public Docs(SinglePackedOrdinals parent, PackedInts.Reader reader) {
+            this.parent = parent;
+            this.reader = reader;
+        }
+
+        @Override
+        public Ordinals ordinals() {
+            return parent;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return parent.getNumDocs();
+        }
+
+        @Override
+        public int getNumOrds() {
+            return parent.getNumOrds();
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getOrd(int docId) {
+            return (int) reader.get(docId);
+        }
+
+        @Override
+        public IntArrayRef getOrds(int docId) {
+            int ordinal = (int) reader.get(docId);
+            if (ordinal == 0) return IntArrayRef.EMPTY;
+            intsScratch.values[0] = docId;
+            return intsScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset((int) reader.get(docId));
+        }
+
+        @Override
+        public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
+            proc.onOrdinal(docId, (int) reader.get(docId));
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
new file mode 100644
index 00000000000..55895305cec
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
@@ -0,0 +1,673 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
+
+    // 0 ordinal in values means no value (its null)
+    private final BytesRef[] values;
+    private final Ordinals ordinals;
+
+    private int[] hashes;
+    private long size = -1;
+
+    public ConcreteBytesRefAtomicFieldData(BytesRef[] values, Ordinals ordinals) {
+        this.values = values;
+        this.ordinals = ordinals;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return ordinals.isMultiValued();
+    }
+
+    @Override
+    public int getNumDocs() {
+        return ordinals.getNumDocs();
+    }
+
+    @Override
+    public boolean isValuesOrdered() {
+        return true;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            long size = RamUsage.NUM_BYTES_ARRAY_HEADER;
+            for (BytesRef value : values) {
+                if (value != null) {
+                    size += RamUsage.NUM_BYTES_OBJECT_REF + RamUsage.NUM_BYTES_OBJECT_HEADER +
+                            RamUsage.NUM_BYTES_ARRAY_HEADER + (value.length + (2 * RamUsage.NUM_BYTES_INT));
+                }
+            }
+            size += ordinals.getMemorySizeInBytes();
+            this.size = size;
+        }
+        return size;
+    }
+
+    @Override
+    public OrdinalsBytesValues getBytesValues() {
+        return ordinals.isMultiValued() ? new BytesValues.Multi(values, ordinals.ordinals()) : new BytesValues.Single(values, ordinals.ordinals());
+    }
+
+    @Override
+    public OrdinalsHashedBytesValues getHashedBytesValues() {
+        if (hashes == null) {
+            int[] hashes = new int[values.length];
+            for (int i = 0; i < values.length; i++) {
+                BytesRef value = values[i];
+                hashes[i] = value == null ? 0 : value.hashCode();
+            }
+            this.hashes = hashes;
+        }
+        return ordinals.isMultiValued() ? new HashedBytesValues.Multi(values, hashes, ordinals.ordinals()) : new HashedBytesValues.Single(values, hashes, ordinals.ordinals());
+    }
+
+    @Override
+    public OrdinalsStringValues getStringValues() {
+        return ordinals.isMultiValued() ? new StringValues.Multi(values, ordinals.ordinals()) : new StringValues.Single(values, ordinals.ordinals());
+    }
+
+    @Override
+    public ScriptDocValues.Strings getScriptValues() {
+        return new ScriptDocValues.Strings(getStringValues());
+    }
+
+    static abstract class BytesValues implements org.elasticsearch.index.fielddata.OrdinalsBytesValues {
+
+        protected final BytesRef[] values;
+        protected final Ordinals.Docs ordinals;
+
+        BytesValues(BytesRef[] values, Ordinals.Docs ordinals) {
+            this.values = values;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return this.ordinals;
+        }
+
+        @Override
+        public BytesRef getValueByOrd(int ord) {
+            return values[ord];
+        }
+
+        @Override
+        public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
+            BytesRef value = values[ord];
+            if (value == null) {
+                ret.length = 0;
+            } else {
+                ret.bytes = value.bytes;
+                ret.offset = value.offset;
+                ret.length = value.offset;
+            }
+            return ret;
+        }
+
+        @Override
+        public BytesRef getSafeValueByOrd(int ord) {
+            return values[ord];
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public BytesRef getValue(int docId) {
+            return values[ordinals.getOrd(docId)];
+        }
+
+        @Override
+        public BytesRef getValueScratch(int docId, BytesRef ret) {
+            BytesRef value = values[ordinals.getOrd(docId)];
+            if (value == null) {
+                ret.length = 0;
+            } else {
+                ret.bytes = value.bytes;
+                ret.offset = value.offset;
+                ret.length = value.length;
+            }
+            return ret;
+        }
+
+        @Override
+        public BytesRef getValueSafe(int docId) {
+            return values[ordinals.getOrd(docId)];
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+            forEachValueInDoc(docId, proc);
+        }
+
+        static class Single extends BytesValues {
+
+            private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public BytesRefArrayRef getValues(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return BytesRefArrayRef.EMPTY;
+                arrayScratch.values[0] = values[ord];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(values[ord]);
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(values[ord]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, values[ord]);
+                }
+            }
+        }
+
+        static class Multi extends BytesValues {
+
+            private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[10], 0);
+            private final ValuesIter iter;
+
+            Multi(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public BytesRefArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return BytesRefArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final BytesRef[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(BytesRef[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public BytesRef next() {
+                    BytesRef value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    static abstract class HashedBytesValues implements org.elasticsearch.index.fielddata.OrdinalsHashedBytesValues {
+
+        protected final BytesRef[] values;
+        protected final int[] hashes;
+        protected final Ordinals.Docs ordinals;
+
+        protected final HashedBytesRef scratch = new HashedBytesRef();
+
+        HashedBytesValues(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
+            this.values = values;
+            this.hashes = hashes;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return this.ordinals;
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            return scratch.reset(values[ord], hashes[ord]);
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return new HashedBytesRef(values[ord], hashes[ord]);
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return scratch.reset(values[ord], hashes[ord]);
+        }
+
+        @Override
+        public HashedBytesRef getValueSafe(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return new HashedBytesRef(values[ord], hashes[ord]);
+        }
+
+        static class Single extends HashedBytesValues {
+
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
+                super(values, hashes, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(scratch.reset(values[ord], hashes[ord]));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(new HashedBytesRef(values[ord], hashes[ord]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
+                }
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
+                }
+            }
+        }
+
+        static class Multi extends HashedBytesValues {
+
+            private final ValuesIter iter;
+            private final SafeValuesIter safeIter;
+
+            Multi(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
+                super(values, hashes, ordinals);
+                this.iter = new ValuesIter(values, hashes);
+                this.safeIter = new SafeValuesIter(values, hashes);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return safeIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final BytesRef[] values;
+                private final int[] hashes;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                private final HashedBytesRef scratch = new HashedBytesRef();
+
+                ValuesIter(BytesRef[] values, int[] hashes) {
+                    this.values = values;
+                    this.hashes = hashes;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public HashedBytesRef next() {
+                    HashedBytesRef value = scratch.reset(values[ord], hashes[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+
+            static class SafeValuesIter implements Iter {
+
+                private final BytesRef[] values;
+                private final int[] hashes;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                SafeValuesIter(BytesRef[] values, int[] hashes) {
+                    this.values = values;
+                    this.hashes = hashes;
+                }
+
+                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public HashedBytesRef next() {
+                    HashedBytesRef value = new HashedBytesRef(values[ord], hashes[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    static abstract class StringValues implements OrdinalsStringValues {
+
+        protected final BytesRef[] values;
+        protected final Ordinals.Docs ordinals;
+
+        protected StringValues(BytesRef[] values, Ordinals.Docs ordinals) {
+            this.values = values;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return ordinals;
+        }
+
+        @Override
+        public String getValueByOrd(int ord) {
+            BytesRef value = values[ord];
+            if (value == null) return null;
+            return value.utf8ToString();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public String getValue(int docId) {
+            BytesRef value = values[ordinals.getOrd(docId)];
+            if (value == null) return null;
+            return value.utf8ToString();
+        }
+
+        static class Single extends StringValues {
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return StringArrayRef.EMPTY;
+                BytesRef value = values[ord];
+                arrayScratch.values[0] = value == null ? null : value.utf8ToString();
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(values[ord].utf8ToString());
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                proc.onValue(docId, values[ord].utf8ToString());
+            }
+        }
+
+        static class Multi extends StringValues {
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[10], 0);
+            private final ValuesIter iter;
+
+            Multi(BytesRef[] values, Ordinals.Docs ordinals) {
+                super(values, ordinals);
+                iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return StringArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    BytesRef value = values[ords.values[i]];
+                    arrayScratch.values[arrayScratch.end++] = value == null ? null : value.utf8ToString();
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    BytesRef value = values[ord];
+                    proc.onValue(docId, value == null ? null : value.utf8ToString());
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements StringValues.Iter {
+
+                private final BytesRef[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(BytesRef[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public String next() {
+                    BytesRef value = values[ord];
+                    ord = ordsIter.next();
+                    return value == null ? null : value.utf8ToString();
+                }
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
new file mode 100644
index 00000000000..4713ff529d3
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class ConcreteBytesRefIndexFieldData extends AbstractIndexFieldData<ConcreteBytesRefAtomicFieldData> implements IndexOrdinalFieldData<ConcreteBytesRefAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
+            return new ConcreteBytesRefIndexFieldData(index, indexSettings, fieldName, type, cache);
+        }
+    }
+
+    public ConcreteBytesRefIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldName, fieldDataType, cache);
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        return true;
+    }
+
+    @Override
+    public ConcreteBytesRefAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public ConcreteBytesRefAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldName());
+        if (terms == null) {
+            return new ConcreteBytesRefAtomicFieldData(new BytesRef[1], new EmptyOrdinals(reader.maxDoc()));
+        }
+
+        long size = terms.size();
+        if (size == -1) {
+            size = 1024;
+        }
+        final ArrayList<BytesRef> values = new ArrayList<BytesRef>((int) size);
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add(null); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add(BytesRef.deepCopyOf(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            // optimizing to DocIdOrdinals which will use less memory removes the "order" notion of ordinals
+            // we can potentially do this, but only for things like facet processes
+            // that don't require order
+//            int[] nativeOrdinals = ordinals.get(0);
+//            boolean allHaveValue = true;
+//            for (int nativeOrdinal : nativeOrdinals) {
+//                if (nativeOrdinal == 0) {
+//                    allHaveValue = false;
+//                    break;
+//                }
+//            }
+//            if (allHaveValue) {
+//                BytesRef[] convertedValues = new BytesRef[values.size()];
+//                for (int i = 1; i < nativeOrdinals.length; i++) {
+//                    convertedValues[i] = values.get(nativeOrdinals[i]);
+//                }
+//                return new ConcreteBytesRefAtomicFieldData(convertedValues, DocIdOrdinals.INSTANCE);
+//            } else {
+//                return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new SingleArrayOrdinals(nativeOrdinals));
+//            }
+            return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new SingleArrayOrdinals(ordinals.get(0), termOrd));
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        // TODO support "missingValue" for sortMissingValue options here...
+        return new BytesRefFieldComparatorSource(this);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
new file mode 100644
index 00000000000..a04d4841c32
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
@@ -0,0 +1,913 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldData {
+
+    protected final double[] values;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public DoubleArrayAtomicFieldData(double[] values, int numDocs) {
+        this.values = values;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    public static class WithOrdinals extends DoubleArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(double[] values, int numDocs, Ordinals ordinals) {
+            super(values, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + +RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, ordinals.ordinals());
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final ValuesIter valuesIter;
+
+            StringValues(double[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.valuesIter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public String getValue(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return null;
+                }
+                return Double.toString(values[ord]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return StringArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = Double.toString(values[ords.values[i]]);
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return valuesIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, Double.toString(values[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final double[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public String next() {
+                    String value = Double.toString(values[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final double[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final ValuesIter iter;
+
+            LongValues(double[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (long) values[ord];
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return LongArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (long) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements LongValues.Iter {
+
+                private final double[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public long next() {
+                    double value = values[ord];
+                    ord = ordsIter.next();
+                    return (long) value;
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final double[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final ValuesIter iter;
+
+            DoubleValues(double[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return values[ord];
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return DoubleArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements DoubleValues.Iter {
+
+                private final double[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public double next() {
+                    double value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    /**
+     * A single valued case, where not all values are "set", so we have a FixedBitSet that
+     * indicates which values have an actual value.
+     */
+    public static class SingleFixedSet extends DoubleArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(double[] values, int numDocs, FixedBitSet set) {
+            super(values, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values, set);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, set);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, set);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] values;
+            private final FixedBitSet set;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(double[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public String getValue(int docId) {
+                if (set.get(docId)) {
+                    return Double.toString(values[docId]);
+                } else {
+                    return null;
+                }
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = Double.toString(values[docId]);
+                    return arrayScratch;
+                } else {
+                    return StringArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(Double.toString(values[docId]));
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, Double.toString(values[docId]));
+                } else {
+                    proc.onMissing(docId);
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final double[] values;
+            private final FixedBitSet set;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(double[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                if (set.get(docId)) {
+                    return (long) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (long) values[docId];
+                    return arrayScratch;
+                } else {
+                    return LongArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((long) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (long) values[docId]);
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final double[] values;
+            private final FixedBitSet set;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(double[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                if (set.get(docId)) {
+                    return values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = values[docId];
+                    return arrayScratch;
+                } else {
+                    return DoubleArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, values[docId]);
+                }
+            }
+        }
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends DoubleArrayAtomicFieldData {
+
+        /**
+         * Note, here, we assume that there is no offset by 1 from docId, so position 0
+         * is the value for docId 0.
+         */
+        public Single(double[] values, int numDocs) {
+            super(values, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] values;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(double[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public String getValue(int docId) {
+                return Double.toString(values[docId]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                arrayScratch.values[0] = Double.toString(values[docId]);
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(Double.toString(values[docId]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, Double.toString(values[docId]));
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final double[] values;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(double[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (long) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((long) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (long) values[docId]);
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final double[] values;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(double[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                return values[docId];
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                arrayScratch.values[0] = values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, values[docId]);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
new file mode 100644
index 00000000000..aeca8032f87
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TDoubleArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArrayAtomicFieldData> implements IndexNumericFieldData<DoubleArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
+            return new DoubleArrayIndexFieldData(index, indexSettings, fieldName, type, cache);
+        }
+    }
+
+    public DoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldName, fieldDataType, cache);
+    }
+
+    @Override
+    public NumericType getNumericType() {
+        return NumericType.DOUBLE;
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public DoubleArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public DoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldName());
+        if (terms == null) {
+            return new DoubleArrayAtomicFieldData.Single(new double[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TDoubleArrayList values = new TDoubleArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add(0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add(FieldCache.NUMERIC_UTILS_DOUBLE_PARSER.parseDouble(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            double[] sValues = new double[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sValues[i] = values.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new DoubleArrayAtomicFieldData.Single(sValues, reader.maxDoc());
+            } else {
+                return new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new DoubleArrayAtomicFieldData.WithOrdinals(values.toArray(new double[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        return new DoubleValuesComparatorSource(this, missingValue);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/ByteArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/ByteArrayRef.java
new file mode 100644
index 00000000000..fb7652b99f4
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/ByteArrayRef.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Bytes;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class ByteArrayRef extends AbstractList<Byte> implements RandomAccess {
+
+    public static final ByteArrayRef EMPTY = new ByteArrayRef(new byte[0]);
+
+    public byte[] values;
+    public int start;
+    public int end;
+
+    public ByteArrayRef(byte[] values) {
+        this(values, 0, values.length);
+    }
+
+    public ByteArrayRef(byte[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public ByteArrayRef(byte[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new byte[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Byte get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Byte)
+                && indexOf(values, (Byte) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Byte) {
+            int i = indexOf(values, (Byte) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Byte) {
+            int i = lastIndexOf(values, (Byte) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Byte set(int index, Byte element) {
+        assert index < size();
+        byte oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof ByteArrayRef) {
+            ByteArrayRef that = (ByteArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Bytes.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(byte[] array, byte target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(byte[] array, byte target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/BytesRefArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/BytesRefArrayRef.java
new file mode 100644
index 00000000000..ac9488e1ed8
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/BytesRefArrayRef.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ */
+public class BytesRefArrayRef {
+    public static final BytesRefArrayRef EMPTY = new BytesRefArrayRef(new BytesRef[0]);
+
+    public BytesRef[] values;
+    public int start;
+    public int end;
+
+    public BytesRefArrayRef(BytesRef[] values) {
+        this(values, 0, values.length);
+    }
+
+    public BytesRefArrayRef(BytesRef[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public BytesRefArrayRef(BytesRef[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new BytesRef[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    public int size() {
+        return end - start;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/DoubleArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/DoubleArrayRef.java
new file mode 100644
index 00000000000..c4a8af56364
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/DoubleArrayRef.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Doubles;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class DoubleArrayRef extends AbstractList<Double> implements RandomAccess {
+
+    public static final DoubleArrayRef EMPTY = new DoubleArrayRef(new double[0]);
+
+    public double[] values;
+    public int start;
+    public int end;
+
+    public DoubleArrayRef(double[] values) {
+        this(values, 0, values.length);
+    }
+
+    public DoubleArrayRef(double[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public DoubleArrayRef(double[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new double[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Double get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Double)
+                && indexOf(values, (Double) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Double) {
+            int i = indexOf(values, (Double) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Double) {
+            int i = lastIndexOf(values, (Double) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Double set(int index, Double element) {
+        assert index < size();
+        double oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof DoubleArrayRef) {
+            DoubleArrayRef that = (DoubleArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Doubles.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(double[] array, double target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(double[] array, double target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/FloatArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/FloatArrayRef.java
new file mode 100644
index 00000000000..d7be02c3288
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/FloatArrayRef.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Floats;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class FloatArrayRef extends AbstractList<Float> implements RandomAccess {
+
+    public static final FloatArrayRef EMPTY = new FloatArrayRef(new float[0]);
+
+    public float[] values;
+    public int start;
+    public int end;
+
+    public FloatArrayRef(float[] values) {
+        this(values, 0, values.length);
+    }
+
+    public FloatArrayRef(float[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public FloatArrayRef(float[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new float[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Float get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Float)
+                && indexOf(values, (Float) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Float) {
+            int i = indexOf(values, (Float) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Float) {
+            int i = lastIndexOf(values, (Float) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Float set(int index, Float element) {
+        assert index < size();
+        float oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof FloatArrayRef) {
+            FloatArrayRef that = (FloatArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Floats.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(float[] array, float target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(float[] array, float target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/IntArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/IntArrayRef.java
new file mode 100644
index 00000000000..e21402485da
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/IntArrayRef.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Ints;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class IntArrayRef extends AbstractList<Integer> implements RandomAccess {
+
+    public static final IntArrayRef EMPTY = new IntArrayRef(new int[0]);
+
+    public int[] values;
+    public int start;
+    public int end;
+
+    public IntArrayRef(int[] values) {
+        this(values, 0, values.length);
+    }
+
+    public IntArrayRef(int[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public IntArrayRef(int[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new int[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Integer get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Integer)
+                && indexOf(values, (Integer) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Integer) {
+            int i = indexOf(values, (Integer) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Integer) {
+            int i = lastIndexOf(values, (Integer) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Integer set(int index, Integer element) {
+        assert index < size();
+        int oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof IntArrayRef) {
+            IntArrayRef that = (IntArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Ints.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(int[] array, int target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(int[] array, int target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/LongArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/LongArrayRef.java
new file mode 100644
index 00000000000..cfa5add1bea
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/LongArrayRef.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Longs;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class LongArrayRef extends AbstractList<Long> implements RandomAccess {
+
+    public static final LongArrayRef EMPTY = new LongArrayRef(new long[0]);
+
+    public long[] values;
+    public int start;
+    public int end;
+
+    public LongArrayRef(long[] values) {
+        this(values, 0, values.length);
+    }
+
+    public LongArrayRef(long[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public LongArrayRef(long[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new long[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Long get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Long)
+                && indexOf(values, (Long) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Long) {
+            int i = indexOf(values, (Long) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Long) {
+            int i = lastIndexOf(values, (Long) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Long set(int index, Long element) {
+        assert index < size();
+        long oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof LongArrayRef) {
+            LongArrayRef that = (LongArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Longs.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(long[] array, long target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(long[] array, long target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/ShortArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/ShortArrayRef.java
new file mode 100644
index 00000000000..aa3cd9c2db6
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/ShortArrayRef.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import com.google.common.primitives.Shorts;
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class ShortArrayRef extends AbstractList<Short> implements RandomAccess {
+
+    public static final ShortArrayRef EMPTY = new ShortArrayRef(new short[0]);
+
+    public short[] values;
+    public int start;
+    public int end;
+
+    public ShortArrayRef(short[] values) {
+        this(values, 0, values.length);
+    }
+
+    public ShortArrayRef(short[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public ShortArrayRef(short[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new short[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public Short get(int index) {
+        assert index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        // Overridden to prevent a ton of boxing
+        return (target instanceof Short)
+                && indexOf(values, (Short) target, start, end) != -1;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Short) {
+            int i = indexOf(values, (Short) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        // Overridden to prevent a ton of boxing
+        if (target instanceof Short) {
+            int i = lastIndexOf(values, (Short) target, start, end);
+            if (i >= 0) {
+                return i - start;
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public Short set(int index, Short element) {
+        assert index < size();
+        short oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof ShortArrayRef) {
+            ShortArrayRef that = (ShortArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (values[start + i] != that.values[that.start + i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + Shorts.hashCode(values[i]);
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+
+    private static int indexOf(short[] array, short target, int start, int end) {
+        for (int i = start; i < end; i++) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private static int lastIndexOf(short[] array, short target, int start, int end) {
+        for (int i = end - 1; i >= start; i--) {
+            if (array[i] == target) {
+                return i;
+            }
+        }
+        return -1;
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/StringArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/StringArrayRef.java
new file mode 100644
index 00000000000..8df3a12d12e
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/StringArrayRef.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import org.apache.lucene.util.ArrayUtil;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class StringArrayRef extends AbstractList<String> implements RandomAccess {
+
+    public static final StringArrayRef EMPTY = new StringArrayRef(new String[0]);
+
+    public String[] values;
+    public int start;
+    public int end;
+
+    public StringArrayRef(String[] values) {
+        this(values, 0, values.length);
+    }
+
+    public StringArrayRef(String[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public StringArrayRef(String[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new String[ArrayUtil.oversize(newLength, 32)];
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public String get(int index) {
+        assert index >= 0 && index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        String sTarget = target.toString();
+        for (int i = start; i < end; i++) {
+            if (values[i].equals(sTarget)) return true;
+        }
+        return false;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        String sTarget = target.toString();
+        for (int i = start; i < end; i++) {
+            if (values[i].equals(sTarget)) return (i - start);
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        String sTarget = target.toString();
+        for (int i = end - 1; i >= start; i--) {
+            if (values[i].equals(sTarget)) return (i - start);
+        }
+        return -1;
+    }
+
+    @Override
+    public String set(int index, String element) {
+        assert index >= 0 && index < size();
+        String oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof StringArrayRef) {
+            StringArrayRef that = (StringArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (!values[start + i].equals(that.values[that.start + i])) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + values[i].hashCode();
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
new file mode 100644
index 00000000000..8f4b2091678
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
@@ -0,0 +1,413 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+
+/**
+ */
+@Test
+public abstract class AbstractFieldDataTests {
+
+    protected IndexFieldDataService ifdService;
+    protected IndexWriter writer;
+    protected AtomicReaderContext readerContext;
+
+    protected abstract FieldDataType getFieldDataType();
+
+    public <IFD extends IndexFieldData> IFD getForField(String fieldName) {
+        return ifdService.getForField(fieldName, getFieldDataType());
+    }
+
+    @BeforeMethod
+    public void setup() throws Exception {
+        ifdService = new IndexFieldDataService(new Index("test"));
+        writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));
+    }
+
+    protected AtomicReaderContext refreshReader() throws Exception {
+        if (readerContext != null) {
+            readerContext.reader().close();
+        }
+        AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(writer, true));
+        readerContext = reader.getContext();
+        return readerContext;
+    }
+
+    @AfterMethod
+    public void tearDown() throws Exception {
+        if (readerContext != null) {
+            readerContext.reader().close();
+        }
+        writer.close();
+        ifdService.clear();
+    }
+
+    public static class BytesValuesVerifierProc implements BytesValues.ValueInDocProc {
+
+        private static final BytesRef MISSING = new BytesRef();
+
+        private final int docId;
+        private final List<BytesRef> expected = new ArrayList<BytesRef>();
+
+        private int idx;
+
+        BytesValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public BytesValuesVerifierProc addExpected(String value) {
+            expected.add(new BytesRef(value));
+            return this;
+        }
+
+        public BytesValuesVerifierProc addExpected(BytesRef value) {
+            expected.add(value);
+            return this;
+        }
+
+        public BytesValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, BytesRef value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class HashedBytesValuesVerifierProc implements HashedBytesValues.ValueInDocProc {
+
+        private static final HashedBytesRef MISSING = new HashedBytesRef();
+
+        private final int docId;
+        private final List<HashedBytesRef> expected = new ArrayList<HashedBytesRef>();
+
+        private int idx;
+
+        HashedBytesValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public HashedBytesValuesVerifierProc addExpected(String value) {
+            expected.add(new HashedBytesRef(value));
+            return this;
+        }
+
+        public HashedBytesValuesVerifierProc addExpected(BytesRef value) {
+            expected.add(new HashedBytesRef(value));
+            return this;
+        }
+
+        public HashedBytesValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, HashedBytesRef value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class StringValuesVerifierProc implements StringValues.ValueInDocProc {
+
+        private static final String MISSING = new String();
+
+        private final int docId;
+        private final List<String> expected = new ArrayList<String>();
+
+        private int idx;
+
+        StringValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public StringValuesVerifierProc addExpected(String value) {
+            expected.add(value);
+            return this;
+        }
+
+        public StringValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, String value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class LongValuesVerifierProc implements LongValues.ValueInDocProc {
+
+        private static final Long MISSING = new Long(0);
+
+        private final int docId;
+        private final List<Long> expected = new ArrayList<Long>();
+
+        private int idx;
+
+        LongValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public LongValuesVerifierProc addExpected(long value) {
+            expected.add(value);
+            return this;
+        }
+
+        public LongValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, long value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class DoubleValuesVerifierProc implements DoubleValues.ValueInDocProc {
+
+        private static final Double MISSING = new Double(0);
+
+        private final int docId;
+        private final List<Double> expected = new ArrayList<Double>();
+
+        private int idx;
+
+        DoubleValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public DoubleValuesVerifierProc addExpected(double value) {
+            expected.add(value);
+            return this;
+        }
+
+        public DoubleValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, double value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class ByteValuesVerifierProc implements ByteValues.ValueInDocProc {
+
+        private static final Byte MISSING = new Byte((byte) 0);
+
+        private final int docId;
+        private final List<Byte> expected = new ArrayList<Byte>();
+
+        private int idx;
+
+        ByteValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public ByteValuesVerifierProc addExpected(byte value) {
+            expected.add(value);
+            return this;
+        }
+
+        public ByteValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, byte value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class ShortValuesVerifierProc implements ShortValues.ValueInDocProc {
+
+        private static final Short MISSING = new Short((byte) 0);
+
+        private final int docId;
+        private final List<Short> expected = new ArrayList<Short>();
+
+        private int idx;
+
+        ShortValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public ShortValuesVerifierProc addExpected(short value) {
+            expected.add(value);
+            return this;
+        }
+
+        public ShortValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, short value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class IntValuesVerifierProc implements IntValues.ValueInDocProc {
+
+        private static final Integer MISSING = new Integer(0);
+
+        private final int docId;
+        private final List<Integer> expected = new ArrayList<Integer>();
+
+        private int idx;
+
+        IntValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public IntValuesVerifierProc addExpected(int value) {
+            expected.add(value);
+            return this;
+        }
+
+        public IntValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, int value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+
+    public static class FloatValuesVerifierProc implements FloatValues.ValueInDocProc {
+
+        private static final Float MISSING = new Float(0);
+
+        private final int docId;
+        private final List<Float> expected = new ArrayList<Float>();
+
+        private int idx;
+
+        FloatValuesVerifierProc(int docId) {
+            this.docId = docId;
+        }
+
+        public FloatValuesVerifierProc addExpected(float value) {
+            expected.add(value);
+            return this;
+        }
+
+        public FloatValuesVerifierProc addMissing() {
+            expected.add(MISSING);
+            return this;
+        }
+
+        @Override
+        public void onValue(int docId, float value) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(value, equalTo(expected.get(idx++)));
+        }
+
+        @Override
+        public void onMissing(int docId) {
+            assertThat(docId, equalTo(this.docId));
+            assertThat(MISSING, sameInstance(expected.get(idx++)));
+        }
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/ConcreteBytesStringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ConcreteBytesStringFieldDataTests.java
new file mode 100644
index 00000000000..4877d30ae65
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ConcreteBytesStringFieldDataTests.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.index.fielddata.FieldDataType;
+import org.testng.annotations.Test;
+
+/**
+ */
+@Test
+public class ConcreteBytesStringFieldDataTests extends StringFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("string", "concrete_bytes", ImmutableMap.<String, String>of());
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java
new file mode 100644
index 00000000000..01cfcb1984c
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.elasticsearch.index.fielddata.FieldDataType;
+
+/**
+ */
+public class DoubleFieldDataTests extends NumericFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("double");
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
new file mode 100644
index 00000000000..e75fb7a6d5c
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
@@ -0,0 +1,1344 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.search.*;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.util.*;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ */
+@Test
+public abstract class NumericFieldDataTests extends StringFieldDataTests {
+
+    protected abstract FieldDataType getFieldDataType();
+
+    protected String one() {
+        return "1.0";
+    }
+
+    protected String two() {
+        return "2.0";
+    }
+
+    protected String three() {
+        return "3.0";
+    }
+
+    protected String four() {
+        return "4.0";
+    }
+
+    @Override
+    protected void fillSingleValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new DoubleField("value", 1.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testSingleValueAllSetNumber() throws Exception {
+        fillSingleValueAllSet();
+        IndexNumericFieldData indexFieldData = getForField("value");
+        AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        LongValues longValues = fieldData.getLongValues();
+
+        assertThat(longValues.isMultiValued(), equalTo(false));
+
+        assertThat(longValues.hasValue(0), equalTo(true));
+        assertThat(longValues.hasValue(1), equalTo(true));
+        assertThat(longValues.hasValue(2), equalTo(true));
+
+        assertThat(longValues.getValue(0), equalTo(2l));
+        assertThat(longValues.getValue(1), equalTo(1l));
+        assertThat(longValues.getValue(2), equalTo(3l));
+
+        assertThat(longValues.getValueMissing(0, -1), equalTo(2l));
+        assertThat(longValues.getValueMissing(1, -1), equalTo(1l));
+        assertThat(longValues.getValueMissing(2, -1), equalTo(3l));
+
+        LongArrayRef longArrayRef = longValues.getValues(0);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(2l));
+
+        longArrayRef = longValues.getValues(1);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(1l));
+
+        longArrayRef = longValues.getValues(2);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(3l));
+
+        LongValues.Iter longValuesIter = longValues.getIter(0);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(2l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(1);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(1l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(2);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(3l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValues.forEachValueInDoc(0, new LongValuesVerifierProc(0).addExpected(2l));
+        longValues.forEachValueInDoc(1, new LongValuesVerifierProc(1).addExpected(1l));
+        longValues.forEachValueInDoc(2, new LongValuesVerifierProc(2).addExpected(3l));
+
+        DoubleValues doubleValues = fieldData.getDoubleValues();
+
+        assertThat(doubleValues.isMultiValued(), equalTo(false));
+
+        assertThat(doubleValues.hasValue(0), equalTo(true));
+        assertThat(doubleValues.hasValue(1), equalTo(true));
+        assertThat(doubleValues.hasValue(2), equalTo(true));
+
+        assertThat(doubleValues.getValue(0), equalTo(2d));
+        assertThat(doubleValues.getValue(1), equalTo(1d));
+        assertThat(doubleValues.getValue(2), equalTo(3d));
+
+        assertThat(doubleValues.getValueMissing(0, -1), equalTo(2d));
+        assertThat(doubleValues.getValueMissing(1, -1), equalTo(1d));
+        assertThat(doubleValues.getValueMissing(2, -1), equalTo(3d));
+
+        DoubleArrayRef doubleArrayRef = doubleValues.getValues(0);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(2d));
+
+        doubleArrayRef = doubleValues.getValues(1);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(1d));
+
+        doubleArrayRef = doubleValues.getValues(2);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(3d));
+
+        DoubleValues.Iter doubleValuesIter = doubleValues.getIter(0);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(2d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(1);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(1d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(2);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(3d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValues.forEachValueInDoc(0, new DoubleValuesVerifierProc(0).addExpected(2d));
+        doubleValues.forEachValueInDoc(1, new DoubleValuesVerifierProc(1).addExpected(1d));
+        doubleValues.forEachValueInDoc(2, new DoubleValuesVerifierProc(2).addExpected(3d));
+
+        ByteValues byteValues = fieldData.getByteValues();
+
+        assertThat(byteValues.isMultiValued(), equalTo(false));
+
+        assertThat(byteValues.hasValue(0), equalTo(true));
+        assertThat(byteValues.hasValue(1), equalTo(true));
+        assertThat(byteValues.hasValue(2), equalTo(true));
+
+        assertThat(byteValues.getValue(0), equalTo((byte) 2));
+        assertThat(byteValues.getValue(1), equalTo((byte) 1));
+        assertThat(byteValues.getValue(2), equalTo((byte) 3));
+
+        assertThat(byteValues.getValueMissing(0, (byte) -1), equalTo((byte) 2));
+        assertThat(byteValues.getValueMissing(1, (byte) -1), equalTo((byte) 1));
+        assertThat(byteValues.getValueMissing(2, (byte) -1), equalTo((byte) 3));
+
+        ByteArrayRef byteArrayRef = byteValues.getValues(0);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 2));
+
+        byteArrayRef = byteValues.getValues(1);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 1));
+
+        byteArrayRef = byteValues.getValues(2);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 3));
+
+        ByteValues.Iter byteValuesIter = byteValues.getIter(0);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 2));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(1);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 1));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(2);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 3));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValues.forEachValueInDoc(0, new ByteValuesVerifierProc(0).addExpected((byte) 2));
+        byteValues.forEachValueInDoc(1, new ByteValuesVerifierProc(1).addExpected((byte) 1));
+        byteValues.forEachValueInDoc(2, new ByteValuesVerifierProc(2).addExpected((byte) 3));
+
+        ShortValues shortValues = fieldData.getShortValues();
+
+        assertThat(shortValues.isMultiValued(), equalTo(false));
+
+        assertThat(shortValues.hasValue(0), equalTo(true));
+        assertThat(shortValues.hasValue(1), equalTo(true));
+        assertThat(shortValues.hasValue(2), equalTo(true));
+
+        assertThat(shortValues.getValue(0), equalTo((short) 2));
+        assertThat(shortValues.getValue(1), equalTo((short) 1));
+        assertThat(shortValues.getValue(2), equalTo((short) 3));
+
+        assertThat(shortValues.getValueMissing(0, (short) -1), equalTo((short) 2));
+        assertThat(shortValues.getValueMissing(1, (short) -1), equalTo((short) 1));
+        assertThat(shortValues.getValueMissing(2, (short) -1), equalTo((short) 3));
+
+        ShortArrayRef shortArrayRef = shortValues.getValues(0);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 2));
+
+        shortArrayRef = shortValues.getValues(1);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 1));
+
+        shortArrayRef = shortValues.getValues(2);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 3));
+
+        ShortValues.Iter shortValuesIter = shortValues.getIter(0);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 2));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(1);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 1));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(2);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 3));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValues.forEachValueInDoc(0, new ShortValuesVerifierProc(0).addExpected((short) 2));
+        shortValues.forEachValueInDoc(1, new ShortValuesVerifierProc(1).addExpected((short) 1));
+        shortValues.forEachValueInDoc(2, new ShortValuesVerifierProc(2).addExpected((short) 3));
+
+        IntValues intValues = fieldData.getIntValues();
+
+        assertThat(intValues.isMultiValued(), equalTo(false));
+
+        assertThat(intValues.hasValue(0), equalTo(true));
+        assertThat(intValues.hasValue(1), equalTo(true));
+        assertThat(intValues.hasValue(2), equalTo(true));
+
+        assertThat(intValues.getValue(0), equalTo(2));
+        assertThat(intValues.getValue(1), equalTo(1));
+        assertThat(intValues.getValue(2), equalTo(3));
+
+        assertThat(intValues.getValueMissing(0, -1), equalTo(2));
+        assertThat(intValues.getValueMissing(1, -1), equalTo(1));
+        assertThat(intValues.getValueMissing(2, -1), equalTo(3));
+
+        IntArrayRef intArrayRef = intValues.getValues(0);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(2));
+
+        intArrayRef = intValues.getValues(1);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(1));
+
+        intArrayRef = intValues.getValues(2);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(3));
+
+        IntValues.Iter intValuesIter = intValues.getIter(0);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(2));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(1);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(1));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(2);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(3));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValues.forEachValueInDoc(0, new IntValuesVerifierProc(0).addExpected(2));
+        intValues.forEachValueInDoc(1, new IntValuesVerifierProc(1).addExpected(1));
+        intValues.forEachValueInDoc(2, new IntValuesVerifierProc(2).addExpected(3));
+
+        FloatValues floatValues = fieldData.getFloatValues();
+
+        assertThat(floatValues.isMultiValued(), equalTo(false));
+
+        assertThat(floatValues.hasValue(0), equalTo(true));
+        assertThat(floatValues.hasValue(1), equalTo(true));
+        assertThat(floatValues.hasValue(2), equalTo(true));
+
+        assertThat(floatValues.getValue(0), equalTo(2f));
+        assertThat(floatValues.getValue(1), equalTo(1f));
+        assertThat(floatValues.getValue(2), equalTo(3f));
+
+        assertThat(floatValues.getValueMissing(0, -1), equalTo(2f));
+        assertThat(floatValues.getValueMissing(1, -1), equalTo(1f));
+        assertThat(floatValues.getValueMissing(2, -1), equalTo(3f));
+
+        FloatArrayRef floatArrayRef = floatValues.getValues(0);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(2f));
+
+        floatArrayRef = floatValues.getValues(1);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(1f));
+
+        floatArrayRef = floatValues.getValues(2);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(3f));
+
+        FloatValues.Iter floatValuesIter = floatValues.getIter(0);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(2f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(1);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(1f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(2);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(3f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValues.forEachValueInDoc(0, new FloatValuesVerifierProc(0).addExpected(2f));
+        floatValues.forEachValueInDoc(1, new FloatValuesVerifierProc(1).addExpected(1f));
+        floatValues.forEachValueInDoc(2, new FloatValuesVerifierProc(2).addExpected(3f));
+
+        IndexSearcher searcher = new IndexSearcher(readerContext.reader());
+        TopFieldDocs topDocs;
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null))));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null), true)));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
+    }
+
+    @Override
+    protected void fillSingleValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testSingleValueWithMissingNumber() throws Exception {
+        fillSingleValueWithMissing();
+        IndexNumericFieldData indexFieldData = getForField("value");
+        AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        LongValues longValues = fieldData.getLongValues();
+
+        assertThat(longValues.isMultiValued(), equalTo(false));
+
+        assertThat(longValues.hasValue(0), equalTo(true));
+        assertThat(longValues.hasValue(1), equalTo(false));
+        assertThat(longValues.hasValue(2), equalTo(true));
+
+        assertThat(longValues.getValue(0), equalTo(2l));
+        assertThat(longValues.getValue(2), equalTo(3l));
+
+        assertThat(longValues.getValueMissing(0, -1), equalTo(2l));
+        assertThat(longValues.getValueMissing(1, -1), equalTo(-1l));
+        assertThat(longValues.getValueMissing(2, -1), equalTo(3l));
+
+        LongArrayRef longArrayRef = longValues.getValues(0);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(2l));
+
+        longArrayRef = longValues.getValues(1);
+        assertThat(longArrayRef.size(), equalTo(0));
+
+        longArrayRef = longValues.getValues(2);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(3l));
+
+        LongValues.Iter longValuesIter = longValues.getIter(0);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(2l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(1);
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(2);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(3l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValues.forEachValueInDoc(0, new LongValuesVerifierProc(0).addExpected(2l));
+        longValues.forEachValueInDoc(1, new LongValuesVerifierProc(1).addMissing());
+        longValues.forEachValueInDoc(2, new LongValuesVerifierProc(2).addExpected(3l));
+
+        DoubleValues doubleValues = fieldData.getDoubleValues();
+
+        assertThat(doubleValues.isMultiValued(), equalTo(false));
+
+        assertThat(doubleValues.hasValue(0), equalTo(true));
+        assertThat(doubleValues.hasValue(1), equalTo(false));
+        assertThat(doubleValues.hasValue(2), equalTo(true));
+
+        assertThat(doubleValues.getValue(0), equalTo(2d));
+        assertThat(doubleValues.getValue(2), equalTo(3d));
+
+        assertThat(doubleValues.getValueMissing(0, -1), equalTo(2d));
+        assertThat(doubleValues.getValueMissing(1, -1), equalTo(-1d));
+        assertThat(doubleValues.getValueMissing(2, -1), equalTo(3d));
+
+        DoubleArrayRef doubleArrayRef = doubleValues.getValues(0);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(2d));
+
+        doubleArrayRef = doubleValues.getValues(1);
+        assertThat(doubleArrayRef.size(), equalTo(0));
+
+        doubleArrayRef = doubleValues.getValues(2);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(3d));
+
+        DoubleValues.Iter doubleValuesIter = doubleValues.getIter(0);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(2d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(1);
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(2);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(3d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValues.forEachValueInDoc(0, new DoubleValuesVerifierProc(0).addExpected(2d));
+        doubleValues.forEachValueInDoc(1, new DoubleValuesVerifierProc(1).addMissing());
+        doubleValues.forEachValueInDoc(2, new DoubleValuesVerifierProc(2).addExpected(3d));
+
+        ByteValues byteValues = fieldData.getByteValues();
+
+        assertThat(byteValues.isMultiValued(), equalTo(false));
+
+        assertThat(byteValues.hasValue(0), equalTo(true));
+        assertThat(byteValues.hasValue(1), equalTo(false));
+        assertThat(byteValues.hasValue(2), equalTo(true));
+
+        assertThat(byteValues.getValue(0), equalTo((byte) 2));
+        assertThat(byteValues.getValue(2), equalTo((byte) 3));
+
+        assertThat(byteValues.getValueMissing(0, (byte) -1), equalTo((byte) 2));
+        assertThat(byteValues.getValueMissing(1, (byte) -1), equalTo((byte) -1));
+        assertThat(byteValues.getValueMissing(2, (byte) -1), equalTo((byte) 3));
+
+        ByteArrayRef byteArrayRef = byteValues.getValues(0);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 2));
+
+        byteArrayRef = byteValues.getValues(1);
+        assertThat(byteArrayRef.size(), equalTo(0));
+
+        byteArrayRef = byteValues.getValues(2);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 3));
+
+        ByteValues.Iter byteValuesIter = byteValues.getIter(0);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 2));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(1);
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(2);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 3));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValues.forEachValueInDoc(0, new ByteValuesVerifierProc(0).addExpected((byte) 2));
+        byteValues.forEachValueInDoc(1, new ByteValuesVerifierProc(1).addMissing());
+        byteValues.forEachValueInDoc(2, new ByteValuesVerifierProc(2).addExpected((byte) 3));
+
+        ShortValues shortValues = fieldData.getShortValues();
+
+        assertThat(shortValues.isMultiValued(), equalTo(false));
+
+        assertThat(shortValues.hasValue(0), equalTo(true));
+        assertThat(shortValues.hasValue(1), equalTo(false));
+        assertThat(shortValues.hasValue(2), equalTo(true));
+
+        assertThat(shortValues.getValue(0), equalTo((short) 2));
+        assertThat(shortValues.getValue(2), equalTo((short) 3));
+
+        assertThat(shortValues.getValueMissing(0, (short) -1), equalTo((short) 2));
+        assertThat(shortValues.getValueMissing(1, (short) -1), equalTo((short) -1));
+        assertThat(shortValues.getValueMissing(2, (short) -1), equalTo((short) 3));
+
+        ShortArrayRef shortArrayRef = shortValues.getValues(0);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 2));
+
+        shortArrayRef = shortValues.getValues(1);
+        assertThat(shortArrayRef.size(), equalTo(0));
+
+        shortArrayRef = shortValues.getValues(2);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 3));
+
+        ShortValues.Iter shortValuesIter = shortValues.getIter(0);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 2));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(1);
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(2);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 3));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValues.forEachValueInDoc(0, new ShortValuesVerifierProc(0).addExpected((short) 2));
+        shortValues.forEachValueInDoc(1, new ShortValuesVerifierProc(1).addMissing());
+        shortValues.forEachValueInDoc(2, new ShortValuesVerifierProc(2).addExpected((short) 3));
+
+        IntValues intValues = fieldData.getIntValues();
+
+        assertThat(intValues.isMultiValued(), equalTo(false));
+
+        assertThat(intValues.hasValue(0), equalTo(true));
+        assertThat(intValues.hasValue(1), equalTo(false));
+        assertThat(intValues.hasValue(2), equalTo(true));
+
+        assertThat(intValues.getValue(0), equalTo(2));
+        assertThat(intValues.getValue(2), equalTo(3));
+
+        assertThat(intValues.getValueMissing(0, -1), equalTo(2));
+        assertThat(intValues.getValueMissing(1, -1), equalTo(-1));
+        assertThat(intValues.getValueMissing(2, -1), equalTo(3));
+
+        IntArrayRef intArrayRef = intValues.getValues(0);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(2));
+
+        intArrayRef = intValues.getValues(1);
+        assertThat(intArrayRef.size(), equalTo(0));
+
+        intArrayRef = intValues.getValues(2);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(3));
+
+        IntValues.Iter intValuesIter = intValues.getIter(0);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(2));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(1);
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(2);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(3));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValues.forEachValueInDoc(0, new IntValuesVerifierProc(0).addExpected(2));
+        intValues.forEachValueInDoc(1, new IntValuesVerifierProc(1).addMissing());
+        intValues.forEachValueInDoc(2, new IntValuesVerifierProc(2).addExpected(3));
+
+        FloatValues floatValues = fieldData.getFloatValues();
+
+        assertThat(floatValues.isMultiValued(), equalTo(false));
+
+        assertThat(floatValues.hasValue(0), equalTo(true));
+        assertThat(floatValues.hasValue(1), equalTo(false));
+        assertThat(floatValues.hasValue(2), equalTo(true));
+
+        assertThat(floatValues.getValue(0), equalTo(2f));
+        assertThat(floatValues.getValue(2), equalTo(3f));
+
+        assertThat(floatValues.getValueMissing(0, -1), equalTo(2f));
+        assertThat(floatValues.getValueMissing(1, -1), equalTo(-1f));
+        assertThat(floatValues.getValueMissing(2, -1), equalTo(3f));
+
+        FloatArrayRef floatArrayRef = floatValues.getValues(0);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(2f));
+
+        floatArrayRef = floatValues.getValues(1);
+        assertThat(floatArrayRef.size(), equalTo(0));
+
+        floatArrayRef = floatValues.getValues(2);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(3f));
+
+        FloatValues.Iter floatValuesIter = floatValues.getIter(0);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(2f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(1);
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(2);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(3f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValues.forEachValueInDoc(0, new FloatValuesVerifierProc(0).addExpected(2f));
+        floatValues.forEachValueInDoc(1, new FloatValuesVerifierProc(1).addMissing());
+        floatValues.forEachValueInDoc(2, new FloatValuesVerifierProc(2).addExpected(3f));
+
+        IndexSearcher searcher = new IndexSearcher(readerContext.reader());
+        TopFieldDocs topDocs;
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null)))); // defaults to _last
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null), true))); // defaults to _last
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource("_first"))));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource("_first"), true)));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(0));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource("1"))));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource("1"), true)));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
+    }
+
+    @Override
+    protected void fillMultiValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        d.add(new DoubleField("value", 4.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new DoubleField("value", 1.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testMultiValueAllSetNumber() throws Exception {
+        fillMultiValueAllSet();
+        IndexNumericFieldData indexFieldData = getForField("value");
+        AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        LongValues longValues = fieldData.getLongValues();
+
+        assertThat(longValues.isMultiValued(), equalTo(true));
+
+        assertThat(longValues.hasValue(0), equalTo(true));
+        assertThat(longValues.hasValue(1), equalTo(true));
+        assertThat(longValues.hasValue(2), equalTo(true));
+
+        assertThat(longValues.getValue(0), equalTo(2l));
+        assertThat(longValues.getValue(1), equalTo(1l));
+        assertThat(longValues.getValue(2), equalTo(3l));
+
+        assertThat(longValues.getValueMissing(0, -1), equalTo(2l));
+        assertThat(longValues.getValueMissing(1, -1), equalTo(1l));
+        assertThat(longValues.getValueMissing(2, -1), equalTo(3l));
+
+        LongArrayRef longArrayRef = longValues.getValues(0);
+        assertThat(longArrayRef.size(), equalTo(2));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(2l));
+        assertThat(longArrayRef.values[longArrayRef.start + 1], equalTo(4l));
+
+        longArrayRef = longValues.getValues(1);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(1l));
+
+        longArrayRef = longValues.getValues(2);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(3l));
+
+        LongValues.Iter longValuesIter = longValues.getIter(0);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(2l));
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(4l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(1);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(1l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(2);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(3l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValues.forEachValueInDoc(0, new LongValuesVerifierProc(0).addExpected(2l).addExpected(4l));
+        longValues.forEachValueInDoc(1, new LongValuesVerifierProc(1).addExpected(1l));
+        longValues.forEachValueInDoc(2, new LongValuesVerifierProc(2).addExpected(3l));
+
+        DoubleValues doubleValues = fieldData.getDoubleValues();
+
+        assertThat(doubleValues.isMultiValued(), equalTo(true));
+
+        assertThat(doubleValues.hasValue(0), equalTo(true));
+        assertThat(doubleValues.hasValue(1), equalTo(true));
+        assertThat(doubleValues.hasValue(2), equalTo(true));
+
+        assertThat(doubleValues.getValue(0), equalTo(2d));
+        assertThat(doubleValues.getValue(1), equalTo(1d));
+        assertThat(doubleValues.getValue(2), equalTo(3d));
+
+        assertThat(doubleValues.getValueMissing(0, -1), equalTo(2d));
+        assertThat(doubleValues.getValueMissing(1, -1), equalTo(1d));
+        assertThat(doubleValues.getValueMissing(2, -1), equalTo(3d));
+
+        DoubleArrayRef doubleArrayRef = doubleValues.getValues(0);
+        assertThat(doubleArrayRef.size(), equalTo(2));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(2d));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start + 1], equalTo(4d));
+
+        doubleArrayRef = doubleValues.getValues(1);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(1d));
+
+        doubleArrayRef = doubleValues.getValues(2);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(3d));
+
+        DoubleValues.Iter doubleValuesIter = doubleValues.getIter(0);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(2d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(4d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(1);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(1d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(2);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(3d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValues.forEachValueInDoc(0, new DoubleValuesVerifierProc(0).addExpected(2d).addExpected(4d));
+        doubleValues.forEachValueInDoc(1, new DoubleValuesVerifierProc(1).addExpected(1d));
+        doubleValues.forEachValueInDoc(2, new DoubleValuesVerifierProc(2).addExpected(3d));
+
+        ByteValues byteValues = fieldData.getByteValues();
+
+        assertThat(byteValues.isMultiValued(), equalTo(true));
+
+        assertThat(byteValues.hasValue(0), equalTo(true));
+        assertThat(byteValues.hasValue(1), equalTo(true));
+        assertThat(byteValues.hasValue(2), equalTo(true));
+
+        assertThat(byteValues.getValue(0), equalTo((byte) 2));
+        assertThat(byteValues.getValue(1), equalTo((byte) 1));
+        assertThat(byteValues.getValue(2), equalTo((byte) 3));
+
+        assertThat(byteValues.getValueMissing(0, (byte) -1), equalTo((byte) 2));
+        assertThat(byteValues.getValueMissing(1, (byte) -1), equalTo((byte) 1));
+        assertThat(byteValues.getValueMissing(2, (byte) -1), equalTo((byte) 3));
+
+        ByteArrayRef byteArrayRef = byteValues.getValues(0);
+        assertThat(byteArrayRef.size(), equalTo(2));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 2));
+        assertThat(byteArrayRef.values[byteArrayRef.start + 1], equalTo((byte) 4));
+
+        byteArrayRef = byteValues.getValues(1);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 1));
+
+        byteArrayRef = byteValues.getValues(2);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 3));
+
+        ByteValues.Iter byteValuesIter = byteValues.getIter(0);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 2));
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 4));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(1);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 1));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(2);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 3));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValues.forEachValueInDoc(0, new ByteValuesVerifierProc(0).addExpected((byte) 2).addExpected((byte) 4));
+        byteValues.forEachValueInDoc(1, new ByteValuesVerifierProc(1).addExpected((byte) 1));
+        byteValues.forEachValueInDoc(2, new ByteValuesVerifierProc(2).addExpected((byte) 3));
+
+        ShortValues shortValues = fieldData.getShortValues();
+
+        assertThat(shortValues.isMultiValued(), equalTo(true));
+
+        assertThat(shortValues.hasValue(0), equalTo(true));
+        assertThat(shortValues.hasValue(1), equalTo(true));
+        assertThat(shortValues.hasValue(2), equalTo(true));
+
+        assertThat(shortValues.getValue(0), equalTo((short) 2));
+        assertThat(shortValues.getValue(1), equalTo((short) 1));
+        assertThat(shortValues.getValue(2), equalTo((short) 3));
+
+        assertThat(shortValues.getValueMissing(0, (short) -1), equalTo((short) 2));
+        assertThat(shortValues.getValueMissing(1, (short) -1), equalTo((short) 1));
+        assertThat(shortValues.getValueMissing(2, (short) -1), equalTo((short) 3));
+
+        ShortArrayRef shortArrayRef = shortValues.getValues(0);
+        assertThat(shortArrayRef.size(), equalTo(2));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 2));
+        assertThat(shortArrayRef.values[shortArrayRef.start + 1], equalTo((short) 4));
+
+        shortArrayRef = shortValues.getValues(1);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 1));
+
+        shortArrayRef = shortValues.getValues(2);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 3));
+
+        ShortValues.Iter shortValuesIter = shortValues.getIter(0);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 2));
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 4));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(1);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 1));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(2);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 3));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValues.forEachValueInDoc(0, new ShortValuesVerifierProc(0).addExpected((short) 2).addExpected((short) 4));
+        shortValues.forEachValueInDoc(1, new ShortValuesVerifierProc(1).addExpected((short) 1));
+        shortValues.forEachValueInDoc(2, new ShortValuesVerifierProc(2).addExpected((short) 3));
+
+        IntValues intValues = fieldData.getIntValues();
+
+        assertThat(intValues.isMultiValued(), equalTo(true));
+
+        assertThat(intValues.hasValue(0), equalTo(true));
+        assertThat(intValues.hasValue(1), equalTo(true));
+        assertThat(intValues.hasValue(2), equalTo(true));
+
+        assertThat(intValues.getValue(0), equalTo(2));
+        assertThat(intValues.getValue(1), equalTo(1));
+        assertThat(intValues.getValue(2), equalTo(3));
+
+        assertThat(intValues.getValueMissing(0, -1), equalTo(2));
+        assertThat(intValues.getValueMissing(1, -1), equalTo(1));
+        assertThat(intValues.getValueMissing(2, -1), equalTo(3));
+
+        IntArrayRef intArrayRef = intValues.getValues(0);
+        assertThat(intArrayRef.size(), equalTo(2));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(2));
+        assertThat(intArrayRef.values[intArrayRef.start + 1], equalTo(4));
+
+        intArrayRef = intValues.getValues(1);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(1));
+
+        intArrayRef = intValues.getValues(2);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(3));
+
+        IntValues.Iter intValuesIter = intValues.getIter(0);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(2));
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(4));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(1);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(1));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(2);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(3));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValues.forEachValueInDoc(0, new IntValuesVerifierProc(0).addExpected(2).addExpected(4));
+        intValues.forEachValueInDoc(1, new IntValuesVerifierProc(1).addExpected(1));
+        intValues.forEachValueInDoc(2, new IntValuesVerifierProc(2).addExpected(3));
+
+        FloatValues floatValues = fieldData.getFloatValues();
+
+        assertThat(floatValues.isMultiValued(), equalTo(true));
+
+        assertThat(floatValues.hasValue(0), equalTo(true));
+        assertThat(floatValues.hasValue(1), equalTo(true));
+        assertThat(floatValues.hasValue(2), equalTo(true));
+
+        assertThat(floatValues.getValue(0), equalTo(2f));
+        assertThat(floatValues.getValue(1), equalTo(1f));
+        assertThat(floatValues.getValue(2), equalTo(3f));
+
+        assertThat(floatValues.getValueMissing(0, -1), equalTo(2f));
+        assertThat(floatValues.getValueMissing(1, -1), equalTo(1f));
+        assertThat(floatValues.getValueMissing(2, -1), equalTo(3f));
+
+        FloatArrayRef floatArrayRef = floatValues.getValues(0);
+        assertThat(floatArrayRef.size(), equalTo(2));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(2f));
+        assertThat(floatArrayRef.values[floatArrayRef.start + 1], equalTo(4f));
+
+        floatArrayRef = floatValues.getValues(1);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(1f));
+
+        floatArrayRef = floatValues.getValues(2);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(3f));
+
+        FloatValues.Iter floatValuesIter = floatValues.getIter(0);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(2f));
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(4f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(1);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(1f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(2);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(3f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValues.forEachValueInDoc(0, new FloatValuesVerifierProc(0).addExpected(2f).addExpected(4f));
+        floatValues.forEachValueInDoc(1, new FloatValuesVerifierProc(1).addExpected(1f));
+        floatValues.forEachValueInDoc(2, new FloatValuesVerifierProc(2).addExpected(3f));
+    }
+
+    @Override
+    protected void fillMultiValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        d.add(new DoubleField("value", 4.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0f, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testMultiValueWithMissingNumber() throws Exception {
+        fillMultiValueWithMissing();
+        IndexNumericFieldData indexFieldData = getForField("value");
+        AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        LongValues longValues = fieldData.getLongValues();
+
+        assertThat(longValues.isMultiValued(), equalTo(true));
+
+        assertThat(longValues.hasValue(0), equalTo(true));
+        assertThat(longValues.hasValue(1), equalTo(false));
+        assertThat(longValues.hasValue(2), equalTo(true));
+
+        assertThat(longValues.getValue(0), equalTo(2l));
+        assertThat(longValues.getValue(2), equalTo(3l));
+
+        assertThat(longValues.getValueMissing(0, -1), equalTo(2l));
+        assertThat(longValues.getValueMissing(1, -1), equalTo(-1l));
+        assertThat(longValues.getValueMissing(2, -1), equalTo(3l));
+
+        LongArrayRef longArrayRef = longValues.getValues(0);
+        assertThat(longArrayRef.size(), equalTo(2));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(2l));
+        assertThat(longArrayRef.values[longArrayRef.start + 1], equalTo(4l));
+
+        longArrayRef = longValues.getValues(1);
+        assertThat(longArrayRef.size(), equalTo(0));
+
+        longArrayRef = longValues.getValues(2);
+        assertThat(longArrayRef.size(), equalTo(1));
+        assertThat(longArrayRef.values[longArrayRef.start], equalTo(3l));
+
+        LongValues.Iter longValuesIter = longValues.getIter(0);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(2l));
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(4l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(1);
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(2);
+        assertThat(longValuesIter.hasNext(), equalTo(true));
+        assertThat(longValuesIter.next(), equalTo(3l));
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValues.forEachValueInDoc(0, new LongValuesVerifierProc(0).addExpected(2l).addExpected(4l));
+        longValues.forEachValueInDoc(1, new LongValuesVerifierProc(1).addMissing());
+        longValues.forEachValueInDoc(2, new LongValuesVerifierProc(2).addExpected(3l));
+
+        DoubleValues doubleValues = fieldData.getDoubleValues();
+
+        assertThat(doubleValues.isMultiValued(), equalTo(true));
+
+        assertThat(doubleValues.hasValue(0), equalTo(true));
+        assertThat(doubleValues.hasValue(1), equalTo(false));
+        assertThat(doubleValues.hasValue(2), equalTo(true));
+
+        assertThat(doubleValues.getValue(0), equalTo(2d));
+        assertThat(doubleValues.getValue(2), equalTo(3d));
+
+        assertThat(doubleValues.getValueMissing(0, -1), equalTo(2d));
+        assertThat(doubleValues.getValueMissing(1, -1), equalTo(-1d));
+        assertThat(doubleValues.getValueMissing(2, -1), equalTo(3d));
+
+        DoubleArrayRef doubleArrayRef = doubleValues.getValues(0);
+        assertThat(doubleArrayRef.size(), equalTo(2));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(2d));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start + 1], equalTo(4d));
+
+        doubleArrayRef = doubleValues.getValues(1);
+        assertThat(doubleArrayRef.size(), equalTo(0));
+
+        doubleArrayRef = doubleValues.getValues(2);
+        assertThat(doubleArrayRef.size(), equalTo(1));
+        assertThat(doubleArrayRef.values[doubleArrayRef.start], equalTo(3d));
+
+        DoubleValues.Iter doubleValuesIter = doubleValues.getIter(0);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(2d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(4d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(1);
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(2);
+        assertThat(doubleValuesIter.hasNext(), equalTo(true));
+        assertThat(doubleValuesIter.next(), equalTo(3d));
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValues.forEachValueInDoc(0, new DoubleValuesVerifierProc(0).addExpected(2d).addExpected(4d));
+        doubleValues.forEachValueInDoc(1, new DoubleValuesVerifierProc(1).addMissing());
+        doubleValues.forEachValueInDoc(2, new DoubleValuesVerifierProc(2).addExpected(3d));
+
+        ByteValues byteValues = fieldData.getByteValues();
+
+        assertThat(byteValues.isMultiValued(), equalTo(true));
+
+        assertThat(byteValues.hasValue(0), equalTo(true));
+        assertThat(byteValues.hasValue(1), equalTo(false));
+        assertThat(byteValues.hasValue(2), equalTo(true));
+
+        assertThat(byteValues.getValue(0), equalTo((byte) 2));
+        assertThat(byteValues.getValue(2), equalTo((byte) 3));
+
+        assertThat(byteValues.getValueMissing(0, (byte) -1), equalTo((byte) 2));
+        assertThat(byteValues.getValueMissing(1, (byte) -1), equalTo((byte) -1));
+        assertThat(byteValues.getValueMissing(2, (byte) -1), equalTo((byte) 3));
+
+        ByteArrayRef byteArrayRef = byteValues.getValues(0);
+        assertThat(byteArrayRef.size(), equalTo(2));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 2));
+        assertThat(byteArrayRef.values[byteArrayRef.start + 1], equalTo((byte) 4));
+
+        byteArrayRef = byteValues.getValues(1);
+        assertThat(byteArrayRef.size(), equalTo(0));
+
+        byteArrayRef = byteValues.getValues(2);
+        assertThat(byteArrayRef.size(), equalTo(1));
+        assertThat(byteArrayRef.values[byteArrayRef.start], equalTo((byte) 3));
+
+        ByteValues.Iter byteValuesIter = byteValues.getIter(0);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 2));
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 4));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(1);
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(2);
+        assertThat(byteValuesIter.hasNext(), equalTo(true));
+        assertThat(byteValuesIter.next(), equalTo((byte) 3));
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValues.forEachValueInDoc(0, new ByteValuesVerifierProc(0).addExpected((byte) 2).addExpected((byte) 4));
+        byteValues.forEachValueInDoc(1, new ByteValuesVerifierProc(1).addMissing());
+        byteValues.forEachValueInDoc(2, new ByteValuesVerifierProc(2).addExpected((byte) 3));
+
+        ShortValues shortValues = fieldData.getShortValues();
+
+        assertThat(shortValues.isMultiValued(), equalTo(true));
+
+        assertThat(shortValues.hasValue(0), equalTo(true));
+        assertThat(shortValues.hasValue(1), equalTo(false));
+        assertThat(shortValues.hasValue(2), equalTo(true));
+
+        assertThat(shortValues.getValue(0), equalTo((short) 2));
+        assertThat(shortValues.getValue(2), equalTo((short) 3));
+
+        assertThat(shortValues.getValueMissing(0, (short) -1), equalTo((short) 2));
+        assertThat(shortValues.getValueMissing(1, (short) -1), equalTo((short) -1));
+        assertThat(shortValues.getValueMissing(2, (short) -1), equalTo((short) 3));
+
+        ShortArrayRef shortArrayRef = shortValues.getValues(0);
+        assertThat(shortArrayRef.size(), equalTo(2));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 2));
+        assertThat(shortArrayRef.values[shortArrayRef.start + 1], equalTo((short) 4));
+
+        shortArrayRef = shortValues.getValues(1);
+        assertThat(shortArrayRef.size(), equalTo(0));
+
+        shortArrayRef = shortValues.getValues(2);
+        assertThat(shortArrayRef.size(), equalTo(1));
+        assertThat(shortArrayRef.values[shortArrayRef.start], equalTo((short) 3));
+
+        ShortValues.Iter shortValuesIter = shortValues.getIter(0);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 2));
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 4));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(1);
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(2);
+        assertThat(shortValuesIter.hasNext(), equalTo(true));
+        assertThat(shortValuesIter.next(), equalTo((short) 3));
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValues.forEachValueInDoc(0, new ShortValuesVerifierProc(0).addExpected((short) 2).addExpected((short) 4));
+        shortValues.forEachValueInDoc(1, new ShortValuesVerifierProc(1).addMissing());
+        shortValues.forEachValueInDoc(2, new ShortValuesVerifierProc(2).addExpected((short) 3));
+
+        IntValues intValues = fieldData.getIntValues();
+
+        assertThat(intValues.isMultiValued(), equalTo(true));
+
+        assertThat(intValues.hasValue(0), equalTo(true));
+        assertThat(intValues.hasValue(1), equalTo(false));
+        assertThat(intValues.hasValue(2), equalTo(true));
+
+        assertThat(intValues.getValue(0), equalTo(2));
+        assertThat(intValues.getValue(2), equalTo(3));
+
+        assertThat(intValues.getValueMissing(0, -1), equalTo(2));
+        assertThat(intValues.getValueMissing(1, -1), equalTo(-1));
+        assertThat(intValues.getValueMissing(2, -1), equalTo(3));
+
+        IntArrayRef intArrayRef = intValues.getValues(0);
+        assertThat(intArrayRef.size(), equalTo(2));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(2));
+        assertThat(intArrayRef.values[intArrayRef.start + 1], equalTo(4));
+
+        intArrayRef = intValues.getValues(1);
+        assertThat(intArrayRef.size(), equalTo(0));
+
+        intArrayRef = intValues.getValues(2);
+        assertThat(intArrayRef.size(), equalTo(1));
+        assertThat(intArrayRef.values[intArrayRef.start], equalTo(3));
+
+        IntValues.Iter intValuesIter = intValues.getIter(0);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(2));
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(4));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(1);
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(2);
+        assertThat(intValuesIter.hasNext(), equalTo(true));
+        assertThat(intValuesIter.next(), equalTo(3));
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValues.forEachValueInDoc(0, new IntValuesVerifierProc(0).addExpected(2).addExpected(4));
+        intValues.forEachValueInDoc(1, new IntValuesVerifierProc(1).addMissing());
+        intValues.forEachValueInDoc(2, new IntValuesVerifierProc(2).addExpected(3));
+
+        FloatValues floatValues = fieldData.getFloatValues();
+
+        assertThat(floatValues.isMultiValued(), equalTo(true));
+
+        assertThat(floatValues.hasValue(0), equalTo(true));
+        assertThat(floatValues.hasValue(1), equalTo(false));
+        assertThat(floatValues.hasValue(2), equalTo(true));
+
+        assertThat(floatValues.getValue(0), equalTo(2f));
+        assertThat(floatValues.getValue(2), equalTo(3f));
+
+        assertThat(floatValues.getValueMissing(0, -1), equalTo(2f));
+        assertThat(floatValues.getValueMissing(1, -1), equalTo(-1f));
+        assertThat(floatValues.getValueMissing(2, -1), equalTo(3f));
+
+        FloatArrayRef floatArrayRef = floatValues.getValues(0);
+        assertThat(floatArrayRef.size(), equalTo(2));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(2f));
+        assertThat(floatArrayRef.values[floatArrayRef.start + 1], equalTo(4f));
+
+        floatArrayRef = floatValues.getValues(1);
+        assertThat(floatArrayRef.size(), equalTo(0));
+
+        floatArrayRef = floatValues.getValues(2);
+        assertThat(floatArrayRef.size(), equalTo(1));
+        assertThat(floatArrayRef.values[floatArrayRef.start], equalTo(3f));
+
+        FloatValues.Iter floatValuesIter = floatValues.getIter(0);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(2f));
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(4f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(1);
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(2);
+        assertThat(floatValuesIter.hasNext(), equalTo(true));
+        assertThat(floatValuesIter.next(), equalTo(3f));
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValues.forEachValueInDoc(0, new FloatValuesVerifierProc(0).addExpected(2f).addExpected(4f));
+        floatValues.forEachValueInDoc(1, new FloatValuesVerifierProc(1).addMissing());
+        floatValues.forEachValueInDoc(2, new FloatValuesVerifierProc(2).addExpected(3f));
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
new file mode 100644
index 00000000000..439329eaec9
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
@@ -0,0 +1,741 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ */
+@Test
+public abstract class StringFieldDataTests extends AbstractFieldDataTests {
+
+    protected String one() {
+        return "1";
+    }
+
+    protected String two() {
+        return "2";
+    }
+
+    protected String three() {
+        return "3";
+    }
+
+    protected String four() {
+        return "4";
+    }
+
+    protected void fillSingleValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "1", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testSingleValueAllSet() throws Exception {
+        fillSingleValueAllSet();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicReaderContext readerContext = refreshReader();
+        AtomicFieldData fieldData = indexFieldData.load(readerContext);
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData.getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(false));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(true));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
+        assertThat(bytesRef, equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(true));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), equalTo(one()));
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(one()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+
+        IndexSearcher searcher = new IndexSearcher(readerContext.reader());
+        TopFieldDocs topDocs;
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null))));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
+
+        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
+                new Sort(new SortField("value", indexFieldData.comparatorSource(null), true)));
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
+    }
+
+    protected void fillSingleValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testSingleValueWithMissing() throws Exception {
+        fillSingleValueWithMissing();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData
+                .getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(false));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(false));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), nullValue());
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), nullValue());
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
+        assertThat(bytesRef, equalTo(new BytesRef()));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(0));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIter(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), nullValue());
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), nullValue());
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIter(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(false));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), nullValue());
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(0));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+
+        // TODO properly support missing....
+    }
+
+    protected void fillMultiValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        d.add(new StringField("value", "4", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new StringField("value", "1", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testMultiValueAllSet() throws Exception {
+        fillMultiValueAllSet();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData.getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(true));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(true));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
+        assertThat(bytesRef, equalTo(new BytesRef(one())));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(2));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(true));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), equalTo(one()));
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(2));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+        assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(four()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(one()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+    }
+
+    protected void fillMultiValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new StringField("value", "2", Field.Store.NO));
+        d.add(new StringField("value", "4", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new StringField("value", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Test
+    public void testMultiValueWithMissing() throws Exception {
+        fillMultiValueWithMissing();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData.getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(true));
+
+        assertThat(bytesValues.hasValue(0), equalTo(true));
+        assertThat(bytesValues.hasValue(1), equalTo(false));
+        assertThat(bytesValues.hasValue(2), equalTo(true));
+
+        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValue(1), nullValue());
+        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
+
+        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueSafe(1), nullValue());
+        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
+        assertThat(bytesRef, equalTo(new BytesRef(two())));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
+        assertThat(bytesRef, equalTo(new BytesRef()));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
+        assertThat(bytesRef, equalTo(new BytesRef(three())));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(2));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(0));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(1));
+        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIter(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(true));
+        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIterSafe(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(true));
+
+        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValue(1), nullValue());
+        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
+
+        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValues.getValueSafe(1), nullValue());
+        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIter(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
+        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(true));
+        assertThat(stringValues.hasValue(1), equalTo(false));
+        assertThat(stringValues.hasValue(2), equalTo(true));
+
+        assertThat(stringValues.getValue(0), equalTo(two()));
+        assertThat(stringValues.getValue(1), nullValue());
+        assertThat(stringValues.getValue(2), equalTo(three()));
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(2));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
+        assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(0));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(1));
+        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(two()));
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(four()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(true));
+        assertThat(stringValuesIter.next(), equalTo(three()));
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
+    }
+}

From 7dc5cf97991b0730bdcbd38802ff8160230c7607 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Fri, 18 Jan 2013 13:17:29 +0100
Subject: [PATCH 02/35] add long field support

---
 .../fielddata/IndexFieldDataService.java      |   7 +-
 .../plain/LongArrayAtomicFieldData.java       | 913 ++++++++++++++++++
 .../plain/LongArrayIndexFieldData.java        | 156 +++
 .../index/fielddata/DoubleFieldDataTests.java |  95 ++
 .../index/fielddata/LongFieldDataTests.java   | 110 +++
 .../fielddata/NumericFieldDataTests.java      |  94 --
 6 files changed, 1279 insertions(+), 96 deletions(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index a536dd3af1e..fa9ce5c775e 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.plain.ConcreteBytesRefIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.LongArrayIndexFieldData;
 import org.elasticsearch.index.settings.IndexSettings;
 
 import java.util.concurrent.ConcurrentMap;
@@ -46,11 +47,13 @@ public class IndexFieldDataService extends AbstractIndexComponent {
         buildersByType = MapBuilder.<String, IndexFieldData.Builder>newMapBuilder()
                 .put("string", new ConcreteBytesRefIndexFieldData.Builder())
                 .put("double", new DoubleArrayIndexFieldData.Builder())
+                .put("long", new LongArrayIndexFieldData.Builder())
                 .immutableMap();
 
-        buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder().
-                put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
+        buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
+                .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
+                .put(Tuple.tuple("long", "array"), new LongArrayIndexFieldData.Builder())
                 .immutableMap();
     }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
new file mode 100644
index 00000000000..dbaa9f90505
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
@@ -0,0 +1,913 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData {
+
+    protected final long[] values;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public LongArrayAtomicFieldData(long[] values, int numDocs) {
+        this.values = values;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    public static class WithOrdinals extends LongArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(long[] values, int numDocs, Ordinals ordinals) {
+            super(values, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + +RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, ordinals.ordinals());
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final long[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final ValuesIter valuesIter;
+
+            StringValues(long[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.valuesIter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public String getValue(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return null;
+                }
+                return Long.toString(values[ord]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return StringArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = Long.toString(values[ords.values[i]]);
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return valuesIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, Long.toString(values[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final long[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(long[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public String next() {
+                    String value = Long.toString(values[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final long[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final ValuesIter iter;
+
+            LongValues(long[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return values[ord];
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return LongArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final long[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(long[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public long next() {
+                    long value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final long[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final ValuesIter iter;
+
+            DoubleValues(long[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (double) values[ord];
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return DoubleArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (double) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (double) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final long[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(long[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public double next() {
+                    double value = (double) values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    /**
+     * A single valued case, where not all values are "set", so we have a FixedBitSet that
+     * indicates which values have an actual value.
+     */
+    public static class SingleFixedSet extends LongArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(long[] values, int numDocs, FixedBitSet set) {
+            super(values, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values, set);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, set);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, set);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final long[] values;
+            private final FixedBitSet set;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(long[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public String getValue(int docId) {
+                if (set.get(docId)) {
+                    return Long.toString(values[docId]);
+                } else {
+                    return null;
+                }
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = Long.toString(values[docId]);
+                    return arrayScratch;
+                } else {
+                    return StringArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(Long.toString(values[docId]));
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, Long.toString(values[docId]));
+                } else {
+                    proc.onMissing(docId);
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final long[] values;
+            private final FixedBitSet set;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(long[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                if (set.get(docId)) {
+                    return values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = values[docId];
+                    return arrayScratch;
+                } else {
+                    return LongArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, values[docId]);
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final long[] values;
+            private final FixedBitSet set;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(long[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                if (set.get(docId)) {
+                    return (double) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (double) values[docId];
+                    return arrayScratch;
+                } else {
+                    return DoubleArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((double) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (double) values[docId]);
+                }
+            }
+        }
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends LongArrayAtomicFieldData {
+
+        /**
+         * Note, here, we assume that there is no offset by 1 from docId, so position 0
+         * is the value for docId 0.
+         */
+        public Single(long[] values, int numDocs) {
+            super(values, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(values);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final long[] values;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(long[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public String getValue(int docId) {
+                return Long.toString(values[docId]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                arrayScratch.values[0] = Long.toString(values[docId]);
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(Long.toString(values[docId]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, Long.toString(values[docId]));
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final long[] values;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(long[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                return values[docId];
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                arrayScratch.values[0] = values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, values[docId]);
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final long[] values;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(long[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (double) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((double) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (double) values[docId]);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
new file mode 100644
index 00000000000..9ac0e310d8f
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TLongArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAtomicFieldData> implements IndexNumericFieldData<LongArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
+            return new LongArrayIndexFieldData(index, indexSettings, fieldName, type, cache);
+        }
+    }
+
+    public LongArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldName, fieldDataType, cache);
+    }
+
+    @Override
+    public NumericType getNumericType() {
+        return NumericType.DOUBLE;
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public LongArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public LongArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldName());
+        if (terms == null) {
+            return new LongArrayAtomicFieldData.Single(new long[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TLongArrayList values = new TLongArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add(0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add(FieldCache.NUMERIC_UTILS_LONG_PARSER.parseLong(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            long[] sValues = new long[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sValues[i] = values.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new LongArrayAtomicFieldData.Single(sValues, reader.maxDoc());
+            } else {
+                return new LongArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new LongArrayAtomicFieldData.WithOrdinals(values.toArray(new long[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        return new DoubleValuesComparatorSource(this, missingValue);
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java
index 01cfcb1984c..02782a75f19 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java
@@ -19,6 +19,10 @@
 
 package org.elasticsearch.test.unit.index.fielddata;
 
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
 import org.elasticsearch.index.fielddata.FieldDataType;
 
 /**
@@ -29,4 +33,95 @@ public class DoubleFieldDataTests extends NumericFieldDataTests {
     protected FieldDataType getFieldDataType() {
         return new FieldDataType("double");
     }
+
+    protected String one() {
+        return "1.0";
+    }
+
+    protected String two() {
+        return "2.0";
+    }
+
+    protected String three() {
+        return "3.0";
+    }
+
+    protected String four() {
+        return "4.0";
+    }
+
+
+    @Override
+    protected void fillSingleValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new DoubleField("value", 1.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillSingleValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        d.add(new DoubleField("value", 4.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new DoubleField("value", 1.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
+        d.add(new DoubleField("value", 4.0d, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new DoubleField("value", 3.0f, Field.Store.NO));
+        writer.addDocument(d);
+    }
 }
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java
new file mode 100644
index 00000000000..6738cfd1b89
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.StringField;
+import org.elasticsearch.index.fielddata.FieldDataType;
+
+/**
+ */
+public class LongFieldDataTests extends NumericFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("long");
+    }
+
+    @Override
+    protected void fillSingleValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new LongField("value", 2, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new LongField("value", 1, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new LongField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillSingleValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new LongField("value", 2, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new LongField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new LongField("value", 2, Field.Store.NO));
+        d.add(new LongField("value", 4, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new LongField("value", 1, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new LongField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new LongField("value", 2, Field.Store.NO));
+        d.add(new LongField("value", 4, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new LongField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
index e75fb7a6d5c..d406f53a1e1 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
@@ -19,10 +19,6 @@
 
 package org.elasticsearch.test.unit.index.fielddata;
 
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
 import org.apache.lucene.search.*;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.util.*;
@@ -38,40 +34,6 @@ public abstract class NumericFieldDataTests extends StringFieldDataTests {
 
     protected abstract FieldDataType getFieldDataType();
 
-    protected String one() {
-        return "1.0";
-    }
-
-    protected String two() {
-        return "2.0";
-    }
-
-    protected String three() {
-        return "3.0";
-    }
-
-    protected String four() {
-        return "4.0";
-    }
-
-    @Override
-    protected void fillSingleValueAllSet() throws Exception {
-        Document d = new Document();
-        d.add(new StringField("_id", "1", Field.Store.NO));
-        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "2", Field.Store.NO));
-        d.add(new DoubleField("value", 1.0d, Field.Store.NO));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "3", Field.Store.NO));
-        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
-        writer.addDocument(d);
-    }
-
     @Test
     public void testSingleValueAllSetNumber() throws Exception {
         fillSingleValueAllSet();
@@ -380,24 +342,6 @@ public abstract class NumericFieldDataTests extends StringFieldDataTests {
         assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
     }
 
-    @Override
-    protected void fillSingleValueWithMissing() throws Exception {
-        Document d = new Document();
-        d.add(new StringField("_id", "1", Field.Store.NO));
-        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "2", Field.Store.NO));
-        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "3", Field.Store.NO));
-        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
-        writer.addDocument(d);
-    }
-
     @Test
     public void testSingleValueWithMissingNumber() throws Exception {
         fillSingleValueWithMissing();
@@ -710,25 +654,6 @@ public abstract class NumericFieldDataTests extends StringFieldDataTests {
         assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
     }
 
-    @Override
-    protected void fillMultiValueAllSet() throws Exception {
-        Document d = new Document();
-        d.add(new StringField("_id", "1", Field.Store.NO));
-        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
-        d.add(new DoubleField("value", 4.0d, Field.Store.NO));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "2", Field.Store.NO));
-        d.add(new DoubleField("value", 1.0d, Field.Store.NO));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "3", Field.Store.NO));
-        d.add(new DoubleField("value", 3.0d, Field.Store.NO));
-        writer.addDocument(d);
-    }
-
     @Test
     public void testMultiValueAllSetNumber() throws Exception {
         fillMultiValueAllSet();
@@ -1038,25 +963,6 @@ public abstract class NumericFieldDataTests extends StringFieldDataTests {
         floatValues.forEachValueInDoc(2, new FloatValuesVerifierProc(2).addExpected(3f));
     }
 
-    @Override
-    protected void fillMultiValueWithMissing() throws Exception {
-        Document d = new Document();
-        d.add(new StringField("_id", "1", Field.Store.NO));
-        d.add(new DoubleField("value", 2.0d, Field.Store.NO));
-        d.add(new DoubleField("value", 4.0d, Field.Store.NO));
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "2", Field.Store.NO));
-        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
-        writer.addDocument(d);
-
-        d = new Document();
-        d.add(new StringField("_id", "3", Field.Store.NO));
-        d.add(new DoubleField("value", 3.0f, Field.Store.NO));
-        writer.addDocument(d);
-    }
-
     @Test
     public void testMultiValueWithMissingNumber() throws Exception {
         fillMultiValueWithMissing();

From e0b280f9b3dc1bf64b35a0d009abe07f3df55686 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Fri, 18 Jan 2013 13:45:10 +0100
Subject: [PATCH 03/35] use FieldMapper.Names for fieldNames, and not just
 fieldName as string

---
 .../index/fielddata/AbstractIndexFieldData.java     | 13 +++++++------
 .../index/fielddata/IndexFieldData.java             |  5 +++--
 .../index/fielddata/IndexFieldDataService.java      | 11 ++++++-----
 .../BytesRefFieldComparatorSource.java              |  2 +-
 .../DoubleValuesComparatorSource.java               |  2 +-
 .../plain/ConcreteBytesRefIndexFieldData.java       | 11 ++++++-----
 .../fielddata/plain/DoubleArrayIndexFieldData.java  | 11 ++++++-----
 .../fielddata/plain/LongArrayIndexFieldData.java    | 11 ++++++-----
 .../index/fielddata/AbstractFieldDataTests.java     |  3 ++-
 9 files changed, 38 insertions(+), 31 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
index 894d98f50fa..d6f6824809f 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
@@ -3,30 +3,31 @@ package org.elasticsearch.index.fielddata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
 /**
  */
 public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
 
-    private final String fieldName;
+    private final FieldMapper.Names fieldNames;
     protected final FieldDataType fieldDataType;
     protected final IndexFieldDataCache cache;
 
-    public AbstractIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+    public AbstractIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
         super(index, indexSettings);
-        this.fieldName = fieldName;
+        this.fieldNames = fieldNames;
         this.fieldDataType = fieldDataType;
         this.cache = cache;
     }
 
     @Override
-    public String getFieldName() {
-        return this.fieldName;
+    public FieldMapper.Names getFieldNames() {
+        return this.fieldNames;
     }
 
     @Override
     public void clear() {
-        cache.clear(index, fieldName);
+        cache.clear(index, fieldNames.indexName());
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index 1b1d1d34dd2..1b74ef9319a 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexComponent;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
 /**
@@ -35,7 +36,7 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
     /**
      * The field name.
      */
-    String getFieldName();
+    FieldMapper.Names getFieldNames();
 
     /**
      * Are the values ordered? (in ascending manner).
@@ -72,6 +73,6 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
 
     interface Builder {
 
-        IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache);
+        IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache);
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index fa9ce5c775e..2ad9274a211 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -32,6 +32,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.plain.ConcreteBytesRefIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.LongArrayIndexFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
 import java.util.concurrent.ConcurrentMap;
@@ -86,7 +87,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
         }
     }
 
-    public <IFD extends IndexFieldData> IFD getForField(String fieldName, FieldDataType type) {
+    public <IFD extends IndexFieldData> IFD getForField(FieldMapper.Names fieldNames, FieldDataType type) {
         IndexFieldData fieldData = loadedFieldData.get(type.getType());
         if (fieldData == null) {
             synchronized (loadedFieldData) {
@@ -100,7 +101,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                         builder = buildersByType.get(type.getType());
                     }
                     if (builder == null) {
-                        throw new ElasticSearchIllegalArgumentException("failed to find field data builder for field " + fieldName + ", and type " + type);
+                        throw new ElasticSearchIllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type);
                     }
 
                     IndexFieldDataCache cache;
@@ -111,14 +112,14 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                         } else if ("soft".equals(cacheType)) {
                             cache = new IndexFieldDataCache.Soft();
                         } else {
-                            throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]");
+                            throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
                         }
                     } else {
                         cache = new IndexFieldDataCache.Resident();
                     }
 
-                    fieldData = builder.build(index, indexSettings, fieldName, type, cache);
-                    loadedFieldData.put(fieldName, fieldData);
+                    fieldData = builder.build(index, indexSettings, fieldNames, type, cache);
+                    loadedFieldData.put(fieldNames.indexName(), fieldData);
                 }
             }
         }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
index e40a090f753..68d436c4455 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
@@ -43,7 +43,7 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat
 
     @Override
     public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-        assert fieldname.equals(indexFieldData.getFieldName());
+        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
         if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexOrdinalFieldData) {
             return new BytesRefOrdValComparator((IndexOrdinalFieldData) indexFieldData, numHits);
         }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
index 84f001f631f..a5692ce44a1 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
@@ -46,7 +46,7 @@ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparato
 
     @Override
     public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-        assert fieldname.equals(indexFieldData.getFieldName());
+        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
 
         double dMissingValue;
         if (missingValue == null || "_last".equals(missingValue)) {
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
index 4713ff529d3..30e0520303a 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
@@ -30,6 +30,7 @@ import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparator
 import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
 import java.util.ArrayList;
@@ -41,13 +42,13 @@ public class ConcreteBytesRefIndexFieldData extends AbstractIndexFieldData<Concr
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
-        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
-            return new ConcreteBytesRefIndexFieldData(index, indexSettings, fieldName, type, cache);
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new ConcreteBytesRefIndexFieldData(index, indexSettings, fieldNames, type, cache);
         }
     }
 
-    public ConcreteBytesRefIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
-        super(index, indexSettings, fieldName, fieldDataType, cache);
+    public ConcreteBytesRefIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
     }
 
     @Override
@@ -72,7 +73,7 @@ public class ConcreteBytesRefIndexFieldData extends AbstractIndexFieldData<Concr
     public ConcreteBytesRefAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
 
-        Terms terms = reader.terms(getFieldName());
+        Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
             return new ConcreteBytesRefAtomicFieldData(new BytesRef[1], new EmptyOrdinals(reader.maxDoc()));
         }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
index aeca8032f87..b6d474dd68a 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
 import java.util.ArrayList;
@@ -42,13 +43,13 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
-        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
-            return new DoubleArrayIndexFieldData(index, indexSettings, fieldName, type, cache);
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new DoubleArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
         }
     }
 
-    public DoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
-        super(index, indexSettings, fieldName, fieldDataType, cache);
+    public DoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
     }
 
     @Override
@@ -80,7 +81,7 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
     public DoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
 
-        Terms terms = reader.terms(getFieldName());
+        Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
             return new DoubleArrayAtomicFieldData.Single(new double[0], 0);
         }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
index 9ac0e310d8f..8e358f875c3 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
 import java.util.ArrayList;
@@ -42,13 +43,13 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
-        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
-            return new LongArrayIndexFieldData(index, indexSettings, fieldName, type, cache);
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new LongArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
         }
     }
 
-    public LongArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
-        super(index, indexSettings, fieldName, fieldDataType, cache);
+    public LongArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
     }
 
     @Override
@@ -80,7 +81,7 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
     public LongArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
 
-        Terms terms = reader.terms(getFieldName());
+        Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
             return new LongArrayAtomicFieldData.Single(new long[0], 0);
         }
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
index 8f4b2091678..8c3aae5ab3a 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.lucene.HashedBytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -50,7 +51,7 @@ public abstract class AbstractFieldDataTests {
     protected abstract FieldDataType getFieldDataType();
 
     public <IFD extends IndexFieldData> IFD getForField(String fieldName) {
-        return ifdService.getForField(fieldName, getFieldDataType());
+        return ifdService.getForField(new FieldMapper.Names(fieldName), getFieldDataType());
     }
 
     @BeforeMethod

From 2d919392534cf78fc89aa176c821147c76bf3089 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Fri, 18 Jan 2013 14:02:39 +0100
Subject: [PATCH 04/35] add initial field data type support to mappers
 hardwired and still happily leaves with current field data impl

---
 .../java/org/elasticsearch/index/mapper/FieldMapper.java    | 2 ++
 .../elasticsearch/index/mapper/core/BinaryFieldMapper.java  | 6 ++++++
 .../elasticsearch/index/mapper/core/BooleanFieldMapper.java | 6 ++++++
 .../elasticsearch/index/mapper/core/ByteFieldMapper.java    | 5 +++++
 .../elasticsearch/index/mapper/core/DateFieldMapper.java    | 5 +++++
 .../elasticsearch/index/mapper/core/DoubleFieldMapper.java  | 5 +++++
 .../elasticsearch/index/mapper/core/FloatFieldMapper.java   | 5 +++++
 .../elasticsearch/index/mapper/core/IntegerFieldMapper.java | 5 +++++
 .../elasticsearch/index/mapper/core/LongFieldMapper.java    | 5 +++++
 .../elasticsearch/index/mapper/core/ShortFieldMapper.java   | 5 +++++
 .../elasticsearch/index/mapper/core/StringFieldMapper.java  | 6 ++++++
 .../elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java | 5 +++++
 .../elasticsearch/index/mapper/internal/AllFieldMapper.java | 6 ++++++
 .../index/mapper/internal/BoostFieldMapper.java             | 6 ++++++
 .../elasticsearch/index/mapper/internal/IdFieldMapper.java  | 6 ++++++
 .../index/mapper/internal/IndexFieldMapper.java             | 6 ++++++
 .../index/mapper/internal/ParentFieldMapper.java            | 6 ++++++
 .../index/mapper/internal/RoutingFieldMapper.java           | 6 ++++++
 .../index/mapper/internal/SourceFieldMapper.java            | 6 ++++++
 .../index/mapper/internal/TypeFieldMapper.java              | 6 ++++++
 .../elasticsearch/index/mapper/internal/UidFieldMapper.java | 6 ++++++
 .../org/elasticsearch/index/mapper/ip/IpFieldMapper.java    | 5 +++++
 22 files changed, 119 insertions(+)

diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index 3898474f378..e0f95d57658 100644
--- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -201,5 +201,7 @@ public interface FieldMapper<T> {
 
     FieldDataType fieldDataType();
 
+    org.elasticsearch.index.fielddata.FieldDataType fieldDataType2();
+
     PostingsFormatProvider postingsFormatProvider();
 }
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
index f8f2f996b3f..01dfcde665b 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.core;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.ElasticSearchParseException;
 import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Strings;
@@ -132,6 +133,11 @@ public class BinaryFieldMapper extends AbstractFieldMapper<BytesReference> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("field data on binary field is not supported");
+    }
+
     @Override
     public Object valueForSearch(Object value) {
         return value(value);
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
index 37aac7933fb..729ddb3f794 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
@@ -23,6 +23,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
@@ -159,6 +160,11 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("not implemented");
+    }
+
     @Override
     public boolean useTermQueryWithQueryString() {
         return true;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
index 3eadbd79883..e62fea95626 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
@@ -128,6 +128,11 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("not implemented");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 32;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 9e5081720a8..70af9552234 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -402,6 +402,11 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
         return FieldDataType.DefaultTypes.LONG;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("not implemented");
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
index ad63da5e24c..3e7ff3c4649 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
@@ -131,6 +131,11 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new org.elasticsearch.index.fielddata.FieldDataType("double");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 64;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
index 79f869f13ad..0b7148a5db5 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
@@ -129,6 +129,11 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("not implemented");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 32;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
index 69aaf18315d..39bf7acd986 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
@@ -131,6 +131,11 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("not implemented");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 32;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
index d9b50f8ea2d..1055acfc93c 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
@@ -131,6 +131,11 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new org.elasticsearch.index.fielddata.FieldDataType("long");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 64;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
index 94378a47375..729aafc6894 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
@@ -131,6 +131,11 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("not implemented");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 32;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
index b73bd997926..6e071572c82 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
@@ -35,6 +35,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NamedCustomAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -211,6 +212,11 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new FieldDataType("string");
+    }
+
     @Override
     public void includeInAll(Boolean includeInAll) {
         if (includeInAll != null) {
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
index ab62fea7239..2d0b4e258bc 100644
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
@@ -147,6 +147,11 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("field data on geo_shape field is not supported");
+    }
+
     @Override
     protected Field parseCreateField(ParseContext context) throws IOException {
         return spatialStrategy.createField(GeoJSONShapeParser.parse(context.parser()));
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
index 9d67fbd48e7..d84d695de7c 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
@@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.all.AllField;
@@ -151,6 +152,11 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("using field data on _all field is not supported");
+    }
+
     @Override
     public Query queryStringTermQuery(Term term) {
         if (!autoBoost) {
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
index fb5919688a4..a5c21775663 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
@@ -27,6 +27,7 @@ import org.apache.lucene.search.NumericRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.Strings;
@@ -131,6 +132,11 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("not implemented");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 32;
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
index c346d53bc15..fe0cf421028 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.lucene.search.XBooleanFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -142,6 +143,11 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new FieldDataType("string");
+    }
+
     @Override
     public String value(Object value) {
         if (value == null) {
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
index e47dbc69127..d278fab1706 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 
@@ -126,6 +127,11 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new FieldDataType("string");
+    }
+
     public String value(Document document) {
         Field field = (Field) document.getField(names.indexName());
         return field == null ? null : value(field);
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
index 6f0cb9e1e43..9eb1f6d96de 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.XTermsFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -134,6 +135,11 @@ public class ParentFieldMapper extends AbstractFieldMapper<Uid> implements Inter
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new FieldDataType("string");
+    }
+
     @Override
     public void preParse(ParseContext context) throws IOException {
     }
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
index 25397949353..c8b81540a6e 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
@@ -129,6 +130,11 @@ public class RoutingFieldMapper extends AbstractFieldMapper<String> implements I
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new FieldDataType("string");
+    }
+
     public void markAsRequired() {
         this.required = true;
     }
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
index 21d7d1ae19c..704f9cb1797 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
@@ -25,6 +25,7 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.ElasticSearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -218,6 +219,11 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        throw new ElasticSearchIllegalArgumentException("field data on _source field is not supported");
+    }
+
     @Override
     public void preParse(ParseContext context) throws IOException {
         super.parse(context);
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index 90a48cd28d8..77283d8a87d 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.lucene.search.TermFilter;
 import org.elasticsearch.common.lucene.search.XConstantScoreQuery;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -109,6 +110,11 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new FieldDataType("string");
+    }
+
     @Override
     public String value(Object value) {
         if (value == null) {
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index 09de0a8ef19..fe95914bb1b 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.UidField;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 
@@ -126,6 +127,11 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new FieldDataType("string");
+    }
+
     @Override
     protected String defaultPostingFormat() {
         return "bloom_default";
diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 44d0d5b7a7f..321991d1d78 100644
--- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -153,6 +153,11 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        return new org.elasticsearch.index.fielddata.FieldDataType("long");
+    }
+
     @Override
     protected int maxPrecisionStep() {
         return 64;

From a39ca58de97bc321d9dedb91ea61a556f7fd43d2 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Fri, 18 Jan 2013 14:12:01 +0100
Subject: [PATCH 05/35] add field data service to index level services

---
 .../index/fielddata/IndexFieldDataModule.java | 39 +++++++++++++++++++
 .../index/service/IndexService.java           |  3 ++
 .../index/service/InternalIndexService.java   | 12 +++++-
 .../indices/InternalIndicesService.java       |  2 +
 .../search/internal/SearchContext.java        |  5 +++
 5 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataModule.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataModule.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataModule.java
new file mode 100644
index 00000000000..8948b0356c5
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataModule.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.settings.Settings;
+
+/**
+ */
+public class IndexFieldDataModule extends AbstractModule {
+
+    private final Settings settings;
+
+    public IndexFieldDataModule(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    protected void configure() {
+        bind(IndexFieldDataService.class).asEagerSingleton();
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/service/IndexService.java b/src/main/java/org/elasticsearch/index/service/IndexService.java
index 6613610292d..de71977ba6c 100644
--- a/src/main/java/org/elasticsearch/index/service/IndexService.java
+++ b/src/main/java/org/elasticsearch/index/service/IndexService.java
@@ -28,6 +28,7 @@ import org.elasticsearch.index.aliases.IndexAliasesService;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.engine.IndexEngine;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.gateway.IndexGateway;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.percolator.PercolatorService;
@@ -48,6 +49,8 @@ public interface IndexService extends IndexComponent, Iterable<IndexShard> {
 
     IndexCache cache();
 
+    IndexFieldDataService fieldData();
+
     IndexSettingsService settingsService();
 
     PercolatorService percolateService();
diff --git a/src/main/java/org/elasticsearch/index/service/InternalIndexService.java b/src/main/java/org/elasticsearch/index/service/InternalIndexService.java
index 35a283babd0..7ce47013959 100644
--- a/src/main/java/org/elasticsearch/index/service/InternalIndexService.java
+++ b/src/main/java/org/elasticsearch/index/service/InternalIndexService.java
@@ -39,6 +39,7 @@ import org.elasticsearch.index.deletionpolicy.DeletionPolicyModule;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineModule;
 import org.elasticsearch.index.engine.IndexEngine;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.gateway.IndexGateway;
 import org.elasticsearch.index.gateway.IndexShardGatewayModule;
 import org.elasticsearch.index.gateway.IndexShardGatewayService;
@@ -112,6 +113,8 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
 
     private final IndexCache indexCache;
 
+    private final IndexFieldDataService indexFieldData;
+
     private final IndexEngine indexEngine;
 
     private final IndexGateway indexGateway;
@@ -130,7 +133,8 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
     public InternalIndexService(Injector injector, Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv, ThreadPool threadPool,
                                 PercolatorService percolatorService, AnalysisService analysisService, MapperService mapperService,
                                 IndexQueryParserService queryParserService, SimilarityService similarityService, IndexAliasesService aliasesService,
-                                IndexCache indexCache, IndexEngine indexEngine, IndexGateway indexGateway, IndexStore indexStore, IndexSettingsService settingsService) {
+                                IndexCache indexCache, IndexEngine indexEngine, IndexGateway indexGateway, IndexStore indexStore, IndexSettingsService settingsService,
+                                IndexFieldDataService indexFieldData) {
         super(index, indexSettings);
         this.injector = injector;
         this.nodeEnv = nodeEnv;
@@ -143,6 +147,7 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
         this.similarityService = similarityService;
         this.aliasesService = aliasesService;
         this.indexCache = indexCache;
+        this.indexFieldData = indexFieldData;
         this.indexEngine = indexEngine;
         this.indexGateway = indexGateway;
         this.indexStore = indexStore;
@@ -211,6 +216,11 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
         return indexCache;
     }
 
+    @Override
+    public IndexFieldDataService fieldData() {
+        return indexFieldData;
+    }
+
     @Override
     public PercolatorService percolateService() {
         return this.percolatorService;
diff --git a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
index f8d71e3cdba..fe24c2e613a 100644
--- a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
+++ b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
@@ -42,6 +42,7 @@ import org.elasticsearch.index.cache.IndexCacheModule;
 import org.elasticsearch.index.codec.CodecModule;
 import org.elasticsearch.index.engine.IndexEngine;
 import org.elasticsearch.index.engine.IndexEngineModule;
+import org.elasticsearch.index.fielddata.IndexFieldDataModule;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.gateway.IndexGateway;
 import org.elasticsearch.index.gateway.IndexGatewayModule;
@@ -276,6 +277,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
         modules.add(new AnalysisModule(indexSettings, indicesAnalysisService));
         modules.add(new SimilarityModule(indexSettings));
         modules.add(new IndexCacheModule(indexSettings));
+        modules.add(new IndexFieldDataModule(indexSettings));
         modules.add(new CodecModule(indexSettings));
         modules.add(new MapperServiceModule());
         modules.add(new IndexQueryParserModule(indexSettings));
diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index f98a89c075f..8706f58f337 100644
--- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -38,6 +38,7 @@ import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.cache.filter.FilterCache;
 import org.elasticsearch.index.cache.id.IdCache;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.index.mapper.MapperService;
@@ -359,6 +360,10 @@ public class SearchContext implements Releasable {
         return indexService.cache().fieldData();
     }
 
+    public IndexFieldDataService fieldData() {
+        return indexService.fieldData();
+    }
+
     public IdCache idCache() {
         return indexService.cache().idCache();
     }

From 692413862a7cd9b3cf5764958a8dbbe075aba37b Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Fri, 18 Jan 2013 14:34:28 +0100
Subject: [PATCH 06/35] add clear when deleting an index for the field data
 service

---
 .../indices/InternalIndicesService.java       |  2 ++
 .../facet/range/RangeFacetProcessor.java      | 33 ++++++++++---------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
index fe24c2e613a..52e0a08c1f3 100644
--- a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
+++ b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
@@ -43,6 +43,7 @@ import org.elasticsearch.index.codec.CodecModule;
 import org.elasticsearch.index.engine.IndexEngine;
 import org.elasticsearch.index.engine.IndexEngineModule;
 import org.elasticsearch.index.fielddata.IndexFieldDataModule;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.gateway.IndexGateway;
 import org.elasticsearch.index.gateway.IndexGatewayModule;
@@ -346,6 +347,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
 
         indexInjector.getInstance(PercolatorService.class).close();
         indexInjector.getInstance(IndexCache.class).close();
+        indexInjector.getInstance(IndexFieldDataService.class).clear();
         indexInjector.getInstance(AnalysisService.class).close();
         indexInjector.getInstance(IndexEngine.class).close();
         indexInjector.getInstance(IndexServiceManagement.class).close();
diff --git a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
index 34b82de09da..0523b78b55a 100644
--- a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.FacetCollector;
@@ -119,22 +120,6 @@ public class RangeFacetProcessor extends AbstractComponent implements FacetProce
 
         RangeFacet.Entry[] rangeEntries = entries.toArray(new RangeFacet.Entry[entries.size()]);
 
-        // fix the range entries if needed
-        if (keyField != null) {
-            FieldMapper mapper = context.smartNameFieldMapper(keyField);
-            if (mapper == null) {
-                throw new FacetPhaseExecutionException(facetName, "No mapping found for key_field [" + keyField + "]");
-            }
-            for (RangeFacet.Entry entry : rangeEntries) {
-                if (entry.fromAsString != null) {
-                    entry.from = ((Number) mapper.value(entry.fromAsString)).doubleValue();
-                }
-                if (entry.toAsString != null) {
-                    entry.to = ((Number) mapper.value(entry.toAsString)).doubleValue();
-                }
-            }
-        }
-
         if (keyScript != null && valueScript != null) {
             return new ScriptRangeFacetCollector(facetName, scriptLang, keyScript, valueScript, params, rangeEntries, context);
         }
@@ -143,6 +128,22 @@ public class RangeFacetProcessor extends AbstractComponent implements FacetProce
             throw new FacetPhaseExecutionException(facetName, "key field is required to be set for range facet, either using [field] or using [key_field]");
         }
 
+        // we have a keyField
+        FieldMapper keyFieldMapper = context.smartNameFieldMapper(keyField);
+        if (keyFieldMapper == null) {
+            throw new FacetPhaseExecutionException(facetName, "No mapping found for key_field [" + keyField + "]");
+        }
+        for (RangeFacet.Entry entry : rangeEntries) {
+            if (entry.fromAsString != null) {
+                entry.from = ((Number) keyFieldMapper.value(entry.fromAsString)).doubleValue();
+            }
+            if (entry.toAsString != null) {
+                entry.to = ((Number) keyFieldMapper.value(entry.toAsString)).doubleValue();
+            }
+        }
+
+        IndexNumericFieldData keyIndexFieldData = context.fieldData().getForField(keyFieldMapper.names(), keyFieldMapper.fieldDataType2());
+
         if (valueField == null || keyField.equals(valueField)) {
             return new RangeFacetCollector(facetName, keyField, rangeEntries, context);
         } else {

From fa363b2dca387e7ac1791c4cfaca32401acb0e4c Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Fri, 18 Jan 2013 14:49:47 +0100
Subject: [PATCH 07/35] move range facet to use new field data abstraction

---
 .../index/mapper/core/DateFieldMapper.java    | 11 +--
 .../range/KeyValueRangeFacetCollector.java    | 74 ++++++-------------
 .../facet/range/RangeFacetCollector.java      | 45 ++++-------
 .../facet/range/RangeFacetProcessor.java      |  9 ++-
 4 files changed, 50 insertions(+), 89 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 70af9552234..045fa403fe5 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -172,6 +172,12 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
         return Defaults.FIELD_TYPE;
     }
 
+    @Override
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        // long for now, need to think about scripts and getting a DateTime back?
+        return new org.elasticsearch.index.fielddata.FieldDataType("long");
+    }
+
     @Override
     protected double parseFuzzyFactor(String fuzzyFactor) {
         if (fuzzyFactor == null) {
@@ -402,11 +408,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
         return FieldDataType.DefaultTypes.LONG;
     }
 
-    @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        throw new ElasticSearchIllegalArgumentException("not implemented");
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/search/facet/range/KeyValueRangeFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/range/KeyValueRangeFacetCollector.java
index cbe237958ac..1817bd93393 100644
--- a/src/main/java/org/elasticsearch/search/facet/range/KeyValueRangeFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/range/KeyValueRangeFacetCollector.java
@@ -20,13 +20,10 @@
 package org.elasticsearch.search.facet.range;
 
 import org.apache.lucene.index.AtomicReaderContext;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -36,53 +33,26 @@ import java.io.IOException;
  */
 public class KeyValueRangeFacetCollector extends AbstractFacetCollector {
 
-    private final String keyIndexFieldName;
-
-    private final String valueIndexFieldName;
-
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType keyFieldDataType;
-    private NumericFieldData keyFieldData;
-
-    private final FieldDataType valueFieldDataType;
+    private final IndexNumericFieldData keyIndexFieldData;
+    private final IndexNumericFieldData valueIndexFieldData;
 
     private final RangeFacet.Entry[] entries;
-
     private final RangeProc rangeProc;
 
-    public KeyValueRangeFacetCollector(String facetName, String keyFieldName, String valueFieldName, RangeFacet.Entry[] entries, SearchContext context) {
+    private DoubleValues keyValues;
+
+    public KeyValueRangeFacetCollector(String facetName, IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, RangeFacet.Entry[] entries, SearchContext context) {
         super(facetName);
         this.entries = entries;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(keyFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + keyFieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        keyIndexFieldName = smartMappers.mapper().names().indexName();
-        keyFieldDataType = smartMappers.mapper().fieldDataType();
-
-        smartMappers = context.smartFieldMappers(valueFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for value_field [" + valueFieldName + "]");
-        }
-        valueIndexFieldName = smartMappers.mapper().names().indexName();
-        valueFieldDataType = smartMappers.mapper().fieldDataType();
-
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueIndexFieldData = valueIndexFieldData;
         this.rangeProc = new RangeProc(entries);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        keyFieldData = (NumericFieldData) fieldDataCache.cache(keyFieldDataType, context.reader(), keyIndexFieldName);
-        rangeProc.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), valueIndexFieldName);
+        keyValues = keyIndexFieldData.load(context).getDoubleValues();
+        rangeProc.valueValues = valueIndexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -90,7 +60,7 @@ public class KeyValueRangeFacetCollector extends AbstractFacetCollector {
         for (RangeFacet.Entry entry : entries) {
             entry.foundInDoc = false;
         }
-        keyFieldData.forEachValueInDoc(doc, rangeProc);
+        keyValues.forEachValueInDoc(doc, rangeProc);
     }
 
     @Override
@@ -98,16 +68,20 @@ public class KeyValueRangeFacetCollector extends AbstractFacetCollector {
         return new InternalRangeFacet(facetName, entries);
     }
 
-    public static class RangeProc implements NumericFieldData.DoubleValueInDocProc {
+    public static class RangeProc implements DoubleValues.ValueInDocProc {
 
         private final RangeFacet.Entry[] entries;
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueValues;
 
         public RangeProc(RangeFacet.Entry[] entries) {
             this.entries = entries;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double value) {
             for (RangeFacet.Entry entry : entries) {
@@ -117,10 +91,9 @@ public class KeyValueRangeFacetCollector extends AbstractFacetCollector {
                 if (value >= entry.getFrom() && value < entry.getTo()) {
                     entry.foundInDoc = true;
                     entry.count++;
-                    if (valueFieldData.multiValued()) {
-                        double[] valuesValues = valueFieldData.doubleValues(docId);
-                        entry.totalCount += valuesValues.length;
-                        for (double valueValue : valuesValues) {
+                    if (valueValues.isMultiValued()) {
+                        for (DoubleValues.Iter iter = valueValues.getIter(docId); iter.hasNext(); ) {
+                            double valueValue = iter.next();
                             entry.total += valueValue;
                             if (valueValue < entry.min) {
                                 entry.min = valueValue;
@@ -128,9 +101,10 @@ public class KeyValueRangeFacetCollector extends AbstractFacetCollector {
                             if (valueValue > entry.max) {
                                 entry.max = valueValue;
                             }
+                            entry.totalCount++;
                         }
-                    } else {
-                        double valueValue = valueFieldData.doubleValue(docId);
+                    } else if (valueValues.hasValue(docId)) {
+                        double valueValue = valueValues.getValue(docId);
                         entry.totalCount++;
                         entry.total += valueValue;
                         if (valueValue < entry.min) {
diff --git a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java
index 1291a344dae..b2bcad6e060 100644
--- a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetCollector.java
@@ -20,13 +20,10 @@
 package org.elasticsearch.search.facet.range;
 
 import org.apache.lucene.index.AtomicReaderContext;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -36,42 +33,22 @@ import java.io.IOException;
  */
 public class RangeFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
-
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private NumericFieldData fieldData;
+    private final IndexNumericFieldData indexFieldData;
 
+    private DoubleValues values;
     private final RangeFacet.Entry[] entries;
-
     private final RangeProc rangeProc;
 
-    public RangeFacetCollector(String facetName, String fieldName, RangeFacet.Entry[] entries, SearchContext context) {
+    public RangeFacetCollector(String facetName, IndexNumericFieldData indexFieldData, RangeFacet.Entry[] entries, SearchContext context) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
+        this.indexFieldData = indexFieldData;
         this.entries = entries;
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        indexFieldName = smartMappers.mapper().names().indexName();
-        fieldDataType = smartMappers.mapper().fieldDataType();
-
         rangeProc = new RangeProc(entries);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -79,7 +56,7 @@ public class RangeFacetCollector extends AbstractFacetCollector {
         for (RangeFacet.Entry entry : entries) {
             entry.foundInDoc = false;
         }
-        fieldData.forEachValueInDoc(doc, rangeProc);
+        values.forEachValueInDoc(doc, rangeProc);
     }
 
     @Override
@@ -87,7 +64,7 @@ public class RangeFacetCollector extends AbstractFacetCollector {
         return new InternalRangeFacet(facetName, entries);
     }
 
-    public static class RangeProc implements NumericFieldData.DoubleValueInDocProc {
+    public static class RangeProc implements DoubleValues.ValueInDocProc {
 
         private final RangeFacet.Entry[] entries;
 
@@ -95,6 +72,10 @@ public class RangeFacetCollector extends AbstractFacetCollector {
             this.entries = entries;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double value) {
             for (RangeFacet.Entry entry : entries) {
diff --git a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
index 0523b78b55a..fdebca926a4 100644
--- a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
@@ -145,10 +145,15 @@ public class RangeFacetProcessor extends AbstractComponent implements FacetProce
         IndexNumericFieldData keyIndexFieldData = context.fieldData().getForField(keyFieldMapper.names(), keyFieldMapper.fieldDataType2());
 
         if (valueField == null || keyField.equals(valueField)) {
-            return new RangeFacetCollector(facetName, keyField, rangeEntries, context);
+            return new RangeFacetCollector(facetName, keyIndexFieldData, rangeEntries, context);
         } else {
+            FieldMapper valueFieldMapper = context.smartNameFieldMapper(valueField);
+            if (valueFieldMapper == null) {
+                throw new FacetPhaseExecutionException(facetName, "No mapping found for value_field [" + keyField + "]");
+            }
+            IndexNumericFieldData valueIndexFieldData = context.fieldData().getForField(valueFieldMapper.names(), valueFieldMapper.fieldDataType2());
             // we have a value field, and its different than the key
-            return new KeyValueRangeFacetCollector(facetName, keyField, valueField, rangeEntries, context);
+            return new KeyValueRangeFacetCollector(facetName, keyIndexFieldData, valueIndexFieldData, rangeEntries, context);
         }
     }
 

From 8c7e0f5ca1bec303b0ffdad76974e56828e5877d Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sat, 19 Jan 2013 08:27:40 +0100
Subject: [PATCH 08/35] fix getOrds on single array ords

---
 .../index/fielddata/ordinals/SingleArrayOrdinals.java           | 2 +-
 .../index/fielddata/ordinals/SinglePackedOrdinals.java          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java
index ae93b8ac2c0..60870d513a0 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java
@@ -112,7 +112,7 @@ public class SingleArrayOrdinals implements Ordinals {
         public IntArrayRef getOrds(int docId) {
             int ordinal = ordinals[docId];
             if (ordinal == 0) return IntArrayRef.EMPTY;
-            intsScratch.values[0] = docId;
+            intsScratch.values[0] = ordinal;
             return intsScratch;
         }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
index 5a3d69253aa..7b46ec122db 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
@@ -116,7 +116,7 @@ public class SinglePackedOrdinals implements Ordinals {
         public IntArrayRef getOrds(int docId) {
             int ordinal = (int) reader.get(docId);
             if (ordinal == 0) return IntArrayRef.EMPTY;
-            intsScratch.values[0] = docId;
+            intsScratch.values[0] = ordinal;
             return intsScratch;
         }
 

From 699ff2782ebdc449e20b1281d7dd8e04d43e9c55 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sat, 19 Jan 2013 10:32:20 +0100
Subject: [PATCH 09/35] move histogram facet to use new field data

---
 .../fielddata/IndexFieldDataService.java      |  4 ++
 .../histogram/HistogramFacetProcessor.java    | 38 ++++++----
 .../BoundedCountHistogramFacetCollector.java  | 46 ++++--------
 .../BoundedValueHistogramFacetCollector.java  | 67 +++++++-----------
 ...dedValueScriptHistogramFacetCollector.java | 45 ++++--------
 .../CountHistogramFacetCollector.java         | 47 ++++---------
 .../FullHistogramFacetCollector.java          | 47 ++++---------
 .../ValueHistogramFacetCollector.java         | 70 ++++++-------------
 .../ValueScriptHistogramFacetCollector.java   | 49 ++++---------
 .../facet/range/RangeFacetProcessor.java      |  4 +-
 10 files changed, 143 insertions(+), 274 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index 2ad9274a211..c942059b0ce 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -87,6 +87,10 @@ public class IndexFieldDataService extends AbstractIndexComponent {
         }
     }
 
+    public <IFD extends IndexFieldData> IFD getForField(FieldMapper mapper) {
+        return getForField(mapper.names(), mapper.fieldDataType2());
+    }
+
     public <IFD extends IndexFieldData> IFD getForField(FieldMapper.Names fieldNames, FieldDataType type) {
         IndexFieldData fieldData = loadedFieldData.get(type.getType());
         if (fieldData == null) {
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/HistogramFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/histogram/HistogramFacetProcessor.java
index 51e51ba3cec..56e8cb5475f 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/HistogramFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/HistogramFacetProcessor.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.FacetCollector;
@@ -115,32 +116,43 @@ public class HistogramFacetProcessor extends AbstractComponent implements FacetP
             throw new FacetPhaseExecutionException(facetName, "[interval] is required to be set for histogram facet");
         }
 
-        if (sFrom != null && sTo != null && keyField != null) {
-            FieldMapper mapper = context.smartNameFieldMapper(keyField);
-            if (mapper == null) {
-                throw new FacetPhaseExecutionException(facetName, "No mapping found for key_field [" + keyField + "]");
+        FieldMapper keyMapper = context.smartNameFieldMapper(keyField);
+        if (keyMapper == null) {
+            throw new FacetPhaseExecutionException(facetName, "No mapping found for key_field [" + keyField + "]");
+        }
+        IndexNumericFieldData keyIndexFieldData = context.fieldData().getForField(keyMapper);
+
+        IndexNumericFieldData valueIndexFieldData = null;
+        if (valueField != null) {
+            FieldMapper valueMapper = context.smartNameFieldMapper(valueField);
+            if (valueMapper == null) {
+                throw new FacetPhaseExecutionException(facetName, "No mapping found for value_field [" + valueField + "]");
             }
-            long from = ((Number) mapper.value(sFrom)).longValue();
-            long to = ((Number) mapper.value(sTo)).longValue();
+            valueIndexFieldData = context.fieldData().getForField(valueMapper);
+        }
+
+        if (sFrom != null && sTo != null && keyField != null) {
+            long from = ((Number) keyMapper.value(sFrom)).longValue();
+            long to = ((Number) keyMapper.value(sTo)).longValue();
 
             if (valueField != null) {
-                return new BoundedValueHistogramFacetCollector(facetName, keyField, valueField, interval, from, to, comparatorType, context);
+                return new BoundedValueHistogramFacetCollector(facetName, keyIndexFieldData, valueIndexFieldData, interval, from, to, comparatorType, context);
             } else if (valueScript != null) {
-                return new BoundedValueScriptHistogramFacetCollector(facetName, keyField, scriptLang, valueScript, params, interval, from, to, comparatorType, context);
+                return new BoundedValueScriptHistogramFacetCollector(facetName, keyIndexFieldData, scriptLang, valueScript, params, interval, from, to, comparatorType, context);
             } else {
-                return new BoundedCountHistogramFacetCollector(facetName, keyField, interval, from, to, comparatorType, context);
+                return new BoundedCountHistogramFacetCollector(facetName, keyIndexFieldData, interval, from, to, comparatorType, context);
             }
         }
 
         if (valueScript != null) {
-            return new ValueScriptHistogramFacetCollector(facetName, keyField, scriptLang, valueScript, params, interval, comparatorType, context);
+            return new ValueScriptHistogramFacetCollector(facetName, keyIndexFieldData, scriptLang, valueScript, params, interval, comparatorType, context);
         } else if (valueField == null) {
-            return new CountHistogramFacetCollector(facetName, keyField, interval, comparatorType, context);
+            return new CountHistogramFacetCollector(facetName, keyIndexFieldData, interval, comparatorType, context);
         } else if (keyField.equals(valueField)) {
-            return new FullHistogramFacetCollector(facetName, keyField, interval, comparatorType, context);
+            return new FullHistogramFacetCollector(facetName, keyIndexFieldData, interval, comparatorType, context);
         } else {
             // we have a value field, and its different than the key
-            return new ValueHistogramFacetCollector(facetName, keyField, valueField, interval, comparatorType, context);
+            return new ValueHistogramFacetCollector(facetName, keyIndexFieldData, valueIndexFieldData, interval, comparatorType, context);
         }
     }
 
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedCountHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedCountHistogramFacetCollector.java
index 1b73ad307cd..659232db31b 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedCountHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedCountHistogramFacetCollector.java
@@ -21,14 +21,10 @@ package org.elasticsearch.search.facet.histogram.bounded;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.histogram.HistogramFacet;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -36,38 +32,18 @@ import java.io.IOException;
 
 public class BoundedCountHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final HistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private NumericFieldData fieldData;
+    private LongValues values;
 
     private final HistogramProc histoProc;
 
-    public BoundedCountHistogramFacetCollector(String facetName, String fieldName, long interval, long from, long to, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public BoundedCountHistogramFacetCollector(String facetName, IndexNumericFieldData indexFieldData, long interval, long from, long to, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        FieldMapper mapper = smartMappers.mapper();
-
-        indexFieldName = mapper.names().indexName();
-        fieldDataType = mapper.fieldDataType();
-
+        this.indexFieldData = indexFieldData;
         long normalizedFrom = (((long) ((double) from / interval)) * interval);
         long normalizedTo = (((long) ((double) to / interval)) * interval);
         if ((to % interval) != 0) {
@@ -81,12 +57,12 @@ public class BoundedCountHistogramFacetCollector extends AbstractFacetCollector
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, histoProc);
+        values.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getLongValues();
     }
 
     @Override
@@ -94,7 +70,7 @@ public class BoundedCountHistogramFacetCollector extends AbstractFacetCollector
         return new InternalBoundedCountHistogramFacet(facetName, comparatorType, histoProc.interval, -histoProc.offset, histoProc.size, histoProc.counts, true);
     }
 
-    public static class HistogramProc implements NumericFieldData.LongValueInDocProc {
+    public static class HistogramProc implements LongValues.ValueInDocProc {
 
         final long from;
         final long to;
@@ -116,6 +92,10 @@ public class BoundedCountHistogramFacetCollector extends AbstractFacetCollector
             this.counts = CacheRecycler.popIntArray(size);
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, long value) {
             if (value <= from || value > to) { // bounds check
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueHistogramFacetCollector.java
index d57cc2974f0..c61ad3b11ca 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueHistogramFacetCollector.java
@@ -21,13 +21,11 @@ package org.elasticsearch.search.facet.histogram.bounded;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.histogram.HistogramFacet;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -38,48 +36,23 @@ import java.io.IOException;
  */
 public class BoundedValueHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String keyIndexFieldName;
-
-    private final String valueIndexFieldName;
+    private final IndexNumericFieldData keyIndexFieldData;
+    private final IndexNumericFieldData valueIndexFieldData;
 
     private final long interval;
 
     private final HistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType keyFieldDataType;
-    private NumericFieldData keyFieldData;
-
-    private final FieldDataType valueFieldDataType;
+    private LongValues keyValues;
 
     private final HistogramProc histoProc;
 
-    public BoundedValueHistogramFacetCollector(String facetName, String keyFieldName, String valueFieldName, long interval, long from, long to, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public BoundedValueHistogramFacetCollector(String facetName, IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, long interval, long from, long to, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.interval = interval;
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(keyFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + keyFieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        keyIndexFieldName = smartMappers.mapper().names().indexName();
-        keyFieldDataType = smartMappers.mapper().fieldDataType();
-
-        smartMappers = context.smartFieldMappers(valueFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for value_field [" + valueFieldName + "]");
-        }
-        valueIndexFieldName = smartMappers.mapper().names().indexName();
-        valueFieldDataType = smartMappers.mapper().fieldDataType();
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueIndexFieldData = valueIndexFieldData;
 
         long normalizedFrom = (((long) ((double) from / interval)) * interval);
         long normalizedTo = (((long) ((double) to / interval)) * interval);
@@ -94,13 +67,13 @@ public class BoundedValueHistogramFacetCollector extends AbstractFacetCollector
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        keyFieldData.forEachValueInDoc(doc, histoProc);
+        keyValues.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        keyFieldData = (NumericFieldData) fieldDataCache.cache(keyFieldDataType, context.reader(), keyIndexFieldName);
-        histoProc.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), valueIndexFieldName);
+        keyValues = keyIndexFieldData.load(context).getLongValues();
+        histoProc.valueValues = valueIndexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -108,7 +81,7 @@ public class BoundedValueHistogramFacetCollector extends AbstractFacetCollector
         return new InternalBoundedFullHistogramFacet(facetName, comparatorType, interval, -histoProc.offset, histoProc.size, histoProc.entries, true);
     }
 
-    public static class HistogramProc implements NumericFieldData.LongValueInDocProc {
+    public static class HistogramProc implements LongValues.ValueInDocProc {
 
         final long from;
         final long to;
@@ -121,7 +94,7 @@ public class BoundedValueHistogramFacetCollector extends AbstractFacetCollector
 
         final Object[] entries;
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueValues;
 
         final ValueAggregator valueAggregator = new ValueAggregator();
 
@@ -134,6 +107,10 @@ public class BoundedValueHistogramFacetCollector extends AbstractFacetCollector
             this.entries = CacheRecycler.popObjectArray(size);
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, long value) {
             if (value <= from || value > to) { // bounds check
@@ -147,14 +124,18 @@ public class BoundedValueHistogramFacetCollector extends AbstractFacetCollector
             }
             entry.count++;
             valueAggregator.entry = entry;
-            valueFieldData.forEachValueInDoc(docId, valueAggregator);
+            valueValues.forEachValueInDoc(docId, valueAggregator);
         }
 
 
-        public static class ValueAggregator implements NumericFieldData.DoubleValueInDocProc {
+        public static class ValueAggregator implements DoubleValues.ValueInDocProc {
 
             InternalBoundedFullHistogramFacet.FullEntry entry;
 
+            @Override
+            public void onMissing(int docId) {
+            }
+
             @Override
             public void onValue(int docId, double value) {
                 entry.totalCount++;
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueScriptHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueScriptHistogramFacetCollector.java
index a19f9bab4fc..2c3e0940c69 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueScriptHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/bounded/BoundedValueScriptHistogramFacetCollector.java
@@ -22,15 +22,11 @@ package org.elasticsearch.search.facet.histogram.bounded;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.histogram.HistogramFacet;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -42,42 +38,23 @@ import java.util.Map;
  */
 public class BoundedValueScriptHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final HistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private NumericFieldData fieldData;
+    private LongValues keyValues;
 
     private final SearchScript valueScript;
 
     private final HistogramProc histoProc;
 
-    public BoundedValueScriptHistogramFacetCollector(String facetName, String fieldName, String scriptLang, String valueScript, Map<String, Object> params, long interval, long from, long to, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public BoundedValueScriptHistogramFacetCollector(String facetName, IndexNumericFieldData indexFieldData, String scriptLang, String valueScript, Map<String, Object> params, long interval, long from, long to, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
+        this.indexFieldData = indexFieldData;
 
         this.valueScript = context.scriptService().search(context.lookup(), scriptLang, valueScript, params);
 
-        FieldMapper mapper = smartMappers.mapper();
-
-        indexFieldName = mapper.names().indexName();
-        fieldDataType = mapper.fieldDataType();
-
         long normalizedFrom = (((long) ((double) from / interval)) * interval);
         long normalizedTo = (((long) ((double) to / interval)) * interval);
         if ((to % interval) != 0) {
@@ -91,7 +68,7 @@ public class BoundedValueScriptHistogramFacetCollector extends AbstractFacetColl
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, histoProc);
+        keyValues.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
@@ -101,7 +78,7 @@ public class BoundedValueScriptHistogramFacetCollector extends AbstractFacetColl
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        keyValues = indexFieldData.load(context).getLongValues();
         valueScript.setNextReader(context);
     }
 
@@ -114,7 +91,7 @@ public class BoundedValueScriptHistogramFacetCollector extends AbstractFacetColl
         return (((long) (value / interval)) * interval);
     }
 
-    public static class HistogramProc implements NumericFieldData.LongValueInDocProc {
+    public static class HistogramProc implements LongValues.ValueInDocProc {
 
         final long from;
         final long to;
@@ -139,6 +116,10 @@ public class BoundedValueScriptHistogramFacetCollector extends AbstractFacetColl
             this.valueScript = valueScript;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, long value) {
             if (value <= from || value > to) { // bounds check
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/CountHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/CountHistogramFacetCollector.java
index f6e8cdbc6d5..2c36711d150 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/CountHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/CountHistogramFacetCollector.java
@@ -22,14 +22,10 @@ package org.elasticsearch.search.facet.histogram.unbounded;
 import gnu.trove.map.hash.TLongLongHashMap;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.histogram.HistogramFacet;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -41,49 +37,28 @@ import java.io.IOException;
  */
 public class CountHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final HistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private NumericFieldData fieldData;
-
+    private DoubleValues values;
     private final HistogramProc histoProc;
 
-    public CountHistogramFacetCollector(String facetName, String fieldName, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public CountHistogramFacetCollector(String facetName, IndexNumericFieldData indexFieldData, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        FieldMapper mapper = smartMappers.mapper();
-
-        indexFieldName = mapper.names().indexName();
-        fieldDataType = mapper.fieldDataType();
-
+        this.indexFieldData = indexFieldData;
         histoProc = new HistogramProc(interval);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, histoProc);
+        values.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -95,7 +70,7 @@ public class CountHistogramFacetCollector extends AbstractFacetCollector {
         return (((long) (value / interval)) * interval);
     }
 
-    public static class HistogramProc implements NumericFieldData.DoubleValueInDocProc {
+    public static class HistogramProc implements DoubleValues.ValueInDocProc {
 
         private final long interval;
 
@@ -105,6 +80,10 @@ public class CountHistogramFacetCollector extends AbstractFacetCollector {
             this.interval = interval;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double value) {
             long bucket = bucket(value, interval);
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java
index f7407e15c86..440523b1854 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/FullHistogramFacetCollector.java
@@ -22,14 +22,10 @@ package org.elasticsearch.search.facet.histogram.unbounded;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.histogram.HistogramFacet;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -41,49 +37,28 @@ import java.io.IOException;
  */
 public class FullHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final HistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private NumericFieldData fieldData;
-
+    private DoubleValues values;
     private final HistogramProc histoProc;
 
-    public FullHistogramFacetCollector(String facetName, String fieldName, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public FullHistogramFacetCollector(String facetName, IndexNumericFieldData indexFieldData, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        FieldMapper mapper = smartMappers.mapper();
-
-        indexFieldName = mapper.names().indexName();
-        fieldDataType = mapper.fieldDataType();
-
+        this.indexFieldData = indexFieldData;
         histoProc = new HistogramProc(interval);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, histoProc);
+        values.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -95,7 +70,7 @@ public class FullHistogramFacetCollector extends AbstractFacetCollector {
         return (((long) (value / interval)) * interval);
     }
 
-    public static class HistogramProc implements NumericFieldData.DoubleValueInDocProc {
+    public static class HistogramProc implements DoubleValues.ValueInDocProc {
 
         final long interval;
 
@@ -105,6 +80,10 @@ public class FullHistogramFacetCollector extends AbstractFacetCollector {
             this.interval = interval;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double value) {
             long bucket = bucket(value, interval);
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java
index 8a3a7428c8b..43560f9ae68 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueHistogramFacetCollector.java
@@ -22,13 +22,10 @@ package org.elasticsearch.search.facet.histogram.unbounded;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.histogram.HistogramFacet;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -39,61 +36,32 @@ import java.io.IOException;
  */
 public class ValueHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String keyIndexFieldName;
-
-    private final String valueIndexFieldName;
-
-    private final long interval;
+    private final IndexNumericFieldData keyIndexFieldData;
+    private final IndexNumericFieldData valueIndexFieldData;
 
     private final HistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType keyFieldDataType;
-    private NumericFieldData keyFieldData;
-
-    private final FieldDataType valueFieldDataType;
+    private DoubleValues keyValues;
 
     private final HistogramProc histoProc;
 
-    public ValueHistogramFacetCollector(String facetName, String keyFieldName, String valueFieldName, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public ValueHistogramFacetCollector(String facetName, IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
-        this.interval = interval;
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(keyFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + keyFieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        keyIndexFieldName = smartMappers.mapper().names().indexName();
-        keyFieldDataType = smartMappers.mapper().fieldDataType();
-
-        smartMappers = context.smartFieldMappers(valueFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for value_field [" + valueFieldName + "]");
-        }
-        valueIndexFieldName = smartMappers.mapper().names().indexName();
-        valueFieldDataType = smartMappers.mapper().fieldDataType();
-
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueIndexFieldData = valueIndexFieldData;
         histoProc = new HistogramProc(interval);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        keyFieldData.forEachValueInDoc(doc, histoProc);
+        keyValues.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        keyFieldData = (NumericFieldData) fieldDataCache.cache(keyFieldDataType, context.reader(), keyIndexFieldName);
-        histoProc.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), valueIndexFieldName);
+        keyValues = keyIndexFieldData.load(context).getDoubleValues();
+        histoProc.valueValues = valueIndexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -101,13 +69,13 @@ public class ValueHistogramFacetCollector extends AbstractFacetCollector {
         return new InternalFullHistogramFacet(facetName, comparatorType, histoProc.entries, true);
     }
 
-    public static class HistogramProc implements NumericFieldData.DoubleValueInDocProc {
+    public static class HistogramProc implements DoubleValues.ValueInDocProc {
 
         final long interval;
 
         final ExtTLongObjectHashMap<InternalFullHistogramFacet.FullEntry> entries = CacheRecycler.popLongObjectMap();
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueValues;
 
         final ValueAggregator valueAggregator = new ValueAggregator();
 
@@ -115,6 +83,10 @@ public class ValueHistogramFacetCollector extends AbstractFacetCollector {
             this.interval = interval;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double value) {
             long bucket = FullHistogramFacetCollector.bucket(value, interval);
@@ -125,13 +97,17 @@ public class ValueHistogramFacetCollector extends AbstractFacetCollector {
             }
             entry.count++;
             valueAggregator.entry = entry;
-            valueFieldData.forEachValueInDoc(docId, valueAggregator);
+            valueValues.forEachValueInDoc(docId, valueAggregator);
         }
 
-        public static class ValueAggregator implements NumericFieldData.DoubleValueInDocProc {
+        public static class ValueAggregator implements DoubleValues.ValueInDocProc {
 
             InternalFullHistogramFacet.FullEntry entry;
 
+            @Override
+            public void onMissing(int docId) {
+            }
+
             @Override
             public void onValue(int docId, double value) {
                 entry.totalCount++;
diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java
index 6a34b417b10..c4bc8f3a68c 100644
--- a/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/histogram/unbounded/ValueScriptHistogramFacetCollector.java
@@ -23,15 +23,11 @@ import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.histogram.HistogramFacet;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -44,48 +40,25 @@ import java.util.Map;
  */
 public class ValueScriptHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final HistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private NumericFieldData fieldData;
-
+    private DoubleValues values;
     private final SearchScript valueScript;
-
     private final HistogramProc histoProc;
 
-    public ValueScriptHistogramFacetCollector(String facetName, String fieldName, String scriptLang, String valueScript, Map<String, Object> params, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public ValueScriptHistogramFacetCollector(String facetName, IndexNumericFieldData indexFieldData, String scriptLang, String valueScript, Map<String, Object> params, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
+        this.indexFieldData = indexFieldData;
         this.valueScript = context.scriptService().search(context.lookup(), scriptLang, valueScript, params);
-
-        FieldMapper mapper = smartMappers.mapper();
-
-        indexFieldName = mapper.names().indexName();
-        fieldDataType = mapper.fieldDataType();
-
         histoProc = new HistogramProc(interval, this.valueScript);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, histoProc);
+        values.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
@@ -95,7 +68,7 @@ public class ValueScriptHistogramFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getDoubleValues();
         valueScript.setNextReader(context);
     }
 
@@ -108,7 +81,7 @@ public class ValueScriptHistogramFacetCollector extends AbstractFacetCollector {
         return (((long) (value / interval)) * interval);
     }
 
-    public static class HistogramProc implements NumericFieldData.DoubleValueInDocProc {
+    public static class HistogramProc implements DoubleValues.ValueInDocProc {
 
         private final long interval;
 
@@ -121,6 +94,10 @@ public class ValueScriptHistogramFacetCollector extends AbstractFacetCollector {
             this.valueScript = valueScript;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double value) {
             valueScript.setNextDocId(docId);
diff --git a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
index fdebca926a4..cb83479afb9 100644
--- a/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/range/RangeFacetProcessor.java
@@ -142,7 +142,7 @@ public class RangeFacetProcessor extends AbstractComponent implements FacetProce
             }
         }
 
-        IndexNumericFieldData keyIndexFieldData = context.fieldData().getForField(keyFieldMapper.names(), keyFieldMapper.fieldDataType2());
+        IndexNumericFieldData keyIndexFieldData = context.fieldData().getForField(keyFieldMapper);
 
         if (valueField == null || keyField.equals(valueField)) {
             return new RangeFacetCollector(facetName, keyIndexFieldData, rangeEntries, context);
@@ -151,7 +151,7 @@ public class RangeFacetProcessor extends AbstractComponent implements FacetProce
             if (valueFieldMapper == null) {
                 throw new FacetPhaseExecutionException(facetName, "No mapping found for value_field [" + keyField + "]");
             }
-            IndexNumericFieldData valueIndexFieldData = context.fieldData().getForField(valueFieldMapper.names(), valueFieldMapper.fieldDataType2());
+            IndexNumericFieldData valueIndexFieldData = context.fieldData().getForField(valueFieldMapper);
             // we have a value field, and its different than the key
             return new KeyValueRangeFacetCollector(facetName, keyIndexFieldData, valueIndexFieldData, rangeEntries, context);
         }

From f1f86efed555e097d4f93122838e1e43ef279a26 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 06:00:57 +0100
Subject: [PATCH 10/35] move statistical facet to use new field data

---
 .../ScriptStatisticalFacetCollector.java      |  5 --
 .../StatisticalFacetCollector.java            | 38 ++++----------
 .../StatisticalFacetProcessor.java            | 19 ++++++-
 .../StatisticalFieldsFacetCollector.java      | 49 +++++--------------
 4 files changed, 38 insertions(+), 73 deletions(-)

diff --git a/src/main/java/org/elasticsearch/search/facet/statistical/ScriptStatisticalFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/statistical/ScriptStatisticalFacetCollector.java
index 6ae5c4eda49..9a97f6caef2 100644
--- a/src/main/java/org/elasticsearch/search/facet/statistical/ScriptStatisticalFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/statistical/ScriptStatisticalFacetCollector.java
@@ -35,15 +35,10 @@ import java.util.Map;
 public class ScriptStatisticalFacetCollector extends AbstractFacetCollector {
 
     private final SearchScript script;
-
     private double min = Double.POSITIVE_INFINITY;
-
     private double max = Double.NEGATIVE_INFINITY;
-
     private double total = 0;
-
     private double sumOfSquares = 0.0;
-
     private long count;
 
     public ScriptStatisticalFacetCollector(String facetName, String scriptLang, String script, Map<String, Object> params, SearchContext context) {
diff --git a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java
index f2777d13a08..23f094ad787 100644
--- a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetCollector.java
@@ -20,13 +20,10 @@
 package org.elasticsearch.search.facet.statistical;
 
 import org.apache.lucene.index.AtomicReaderContext;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -36,42 +33,25 @@ import java.io.IOException;
  */
 public class StatisticalFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private NumericFieldData fieldData;
+    private DoubleValues values;
 
     private final StatsProc statsProc = new StatsProc();
 
-    public StatisticalFacetCollector(String facetName, String fieldName, SearchContext context) {
+    public StatisticalFacetCollector(String facetName, IndexNumericFieldData indexFieldData, SearchContext context) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        indexFieldName = smartMappers.mapper().names().indexName();
-        fieldDataType = smartMappers.mapper().fieldDataType();
+        this.indexFieldData = indexFieldData;
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, statsProc);
+        values.forEachValueInDoc(doc, statsProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -79,7 +59,7 @@ public class StatisticalFacetCollector extends AbstractFacetCollector {
         return new InternalStatisticalFacet(facetName, statsProc.min(), statsProc.max(), statsProc.total(), statsProc.sumOfSquares(), statsProc.count());
     }
 
-    public static class StatsProc implements NumericFieldData.MissingDoubleValueInDocProc {
+    public static class StatsProc implements DoubleValues.ValueInDocProc {
 
         double min = Double.POSITIVE_INFINITY;
 
diff --git a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java
index e51de35a7eb..62e9b84ed45 100644
--- a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java
@@ -24,6 +24,8 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.FacetCollector;
 import org.elasticsearch.search.facet.FacetPhaseExecutionException;
@@ -87,13 +89,26 @@ public class StatisticalFacetProcessor extends AbstractComponent implements Face
             }
         }
         if (fieldsNames != null) {
-            return new StatisticalFieldsFacetCollector(facetName, fieldsNames, context);
+            IndexNumericFieldData[] indexFieldDatas = new IndexNumericFieldData[fieldsNames.length];
+            for (int i = 0; i < fieldsNames.length; i++) {
+                FieldMapper fieldMapper = context.mapperService().smartNameFieldMapper(fieldsNames[i]);
+                if (fieldMapper == null) {
+                    throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldsNames[i] + "]");
+                }
+                indexFieldDatas[i] = context.fieldData().getForField(fieldMapper);
+            }
+            return new StatisticalFieldsFacetCollector(facetName, indexFieldDatas, context);
         }
         if (script == null && field == null) {
             throw new FacetPhaseExecutionException(facetName, "statistical facet requires either [script] or [field] to be set");
         }
         if (field != null) {
-            return new StatisticalFacetCollector(facetName, field, context);
+            FieldMapper fieldMapper = context.mapperService().smartNameFieldMapper(field);
+            if (fieldMapper == null) {
+                throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + field + "]");
+            }
+            IndexNumericFieldData indexFieldData = context.fieldData().getForField(fieldMapper);
+            return new StatisticalFacetCollector(facetName, indexFieldData, context);
         } else {
             return new ScriptStatisticalFacetCollector(facetName, scriptLang, script, params, context);
         }
diff --git a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java
index f2b74bdf67e..df95b0b2987 100644
--- a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFieldsFacetCollector.java
@@ -20,13 +20,10 @@
 package org.elasticsearch.search.facet.statistical;
 
 import org.apache.lucene.index.AtomicReaderContext;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -36,46 +33,29 @@ import java.io.IOException;
  */
 public class StatisticalFieldsFacetCollector extends AbstractFacetCollector {
 
-    private final String[] indexFieldsNames;
+    private final IndexNumericFieldData[] indexFieldDatas;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType[] fieldsDataType;
-
-    private NumericFieldData[] fieldsData;
+    private DoubleValues[] values;
 
     private final StatsProc statsProc = new StatsProc();
 
-    public StatisticalFieldsFacetCollector(String facetName, String[] fieldsNames, SearchContext context) {
+    public StatisticalFieldsFacetCollector(String facetName, IndexNumericFieldData[] indexFieldDatas, SearchContext context) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-
-        fieldsDataType = new FieldDataType[fieldsNames.length];
-        fieldsData = new NumericFieldData[fieldsNames.length];
-        indexFieldsNames = new String[fieldsNames.length];
-
-
-        for (int i = 0; i < fieldsNames.length; i++) {
-            FieldMapper mapper = context.smartNameFieldMapper(fieldsNames[i]);
-            if (mapper == null) {
-                throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldsNames[i] + "]");
-            }
-            indexFieldsNames[i] = mapper.names().indexName();
-            fieldsDataType[i] = mapper.fieldDataType();
-        }
+        this.indexFieldDatas = indexFieldDatas;
+        this.values = new DoubleValues[indexFieldDatas.length];
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        for (NumericFieldData fieldData : fieldsData) {
-            fieldData.forEachValueInDoc(doc, statsProc);
+        for (DoubleValues value : values) {
+            value.forEachValueInDoc(doc, statsProc);
         }
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        for (int i = 0; i < indexFieldsNames.length; i++) {
-            fieldsData[i] = (NumericFieldData) fieldDataCache.cache(fieldsDataType[i], context.reader(), indexFieldsNames[i]);
+        for (int i = 0; i < indexFieldDatas.length; i++) {
+            values[i] = indexFieldDatas[i].load(context).getDoubleValues();
         }
     }
 
@@ -84,18 +64,13 @@ public class StatisticalFieldsFacetCollector extends AbstractFacetCollector {
         return new InternalStatisticalFacet(facetName, statsProc.min(), statsProc.max(), statsProc.total(), statsProc.sumOfSquares(), statsProc.count());
     }
 
-    public static class StatsProc implements NumericFieldData.MissingDoubleValueInDocProc {
+    public static class StatsProc implements DoubleValues.ValueInDocProc {
 
         double min = Double.POSITIVE_INFINITY;
-
         double max = Double.NEGATIVE_INFINITY;
-
         double total = 0;
-
         double sumOfSquares = 0.0;
-
         long count;
-
         int missing;
 
         @Override

From 37acba1b57e5775eaa7f04aba814e1d70936adf1 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 07:10:50 +0100
Subject: [PATCH 11/35] terms stats to use new field data

---
 .../common/lucene/HashedBytesRef.java         | 10 ++
 .../fielddata/IndexNumericFieldData.java      | 44 +++++++--
 .../plain/LongArrayIndexFieldData.java        |  2 +-
 .../termsstats/TermsStatsFacetProcessor.java  | 41 ++++----
 .../TermsStatsDoubleFacetCollector.java       | 75 +++++---------
 .../longs/TermsStatsLongFacetCollector.java   | 81 +++++-----------
 .../InternalTermsStatsStringFacet.java        |  7 +-
 .../TermsStatsStringFacetCollector.java       | 97 +++++++------------
 8 files changed, 158 insertions(+), 199 deletions(-)

diff --git a/src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java b/src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java
index 299de543e9a..44873602d2f 100644
--- a/src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java
+++ b/src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java
@@ -73,4 +73,14 @@ public class HashedBytesRef {
     public String toString() {
         return bytes.toString();
     }
+
+    public HashedBytesRef deepCopy() {
+        return deepCopyOf(this);
+    }
+
+    public static HashedBytesRef deepCopyOf(HashedBytesRef other) {
+        BytesRef copy = new BytesRef();
+        copy.copyBytes(other.bytes);
+        return new HashedBytesRef(copy, other.hash);
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
index 90633cba92a..41873549e87 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
@@ -26,12 +26,44 @@ import org.apache.lucene.index.AtomicReaderContext;
 public interface IndexNumericFieldData<FD extends AtomicNumericFieldData> extends IndexFieldData<FD> {
 
     static enum NumericType {
-        BYTE,
-        SHORT,
-        INT,
-        LONG,
-        FLOAT,
-        DOUBLE
+        BYTE {
+            @Override
+            public boolean isFloatingPoint() {
+                return false;
+            }
+        },
+        SHORT {
+            @Override
+            public boolean isFloatingPoint() {
+                return false;
+            }
+        },
+        INT {
+            @Override
+            public boolean isFloatingPoint() {
+                return false;
+            }
+        },
+        LONG {
+            @Override
+            public boolean isFloatingPoint() {
+                return false;
+            }
+        },
+        FLOAT {
+            @Override
+            public boolean isFloatingPoint() {
+                return true;
+            }
+        },
+        DOUBLE {
+            @Override
+            public boolean isFloatingPoint() {
+                return true;
+            }
+        };
+
+        public abstract boolean isFloatingPoint();
     }
 
     NumericType getNumericType();
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
index 8e358f875c3..b38ead65e84 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
@@ -54,7 +54,7 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
 
     @Override
     public NumericType getNumericType() {
-        return NumericType.DOUBLE;
+        return NumericType.LONG;
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacetProcessor.java
index 9b9de255471..fdd9026ff96 100644
--- a/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/termsstats/TermsStatsFacetProcessor.java
@@ -23,8 +23,10 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.FacetCollector;
 import org.elasticsearch.search.facet.FacetPhaseExecutionException;
@@ -102,24 +104,29 @@ public class TermsStatsFacetProcessor extends AbstractComponent implements Facet
             throw new FacetPhaseExecutionException(facetName, "either [value_field] or [script] are required to be set for terms stats facet");
         }
 
-        FieldMapper keyFieldMapper = context.smartNameFieldMapper(keyField);
-        if (keyFieldMapper != null) {
-            if (keyFieldMapper.fieldDataType() == FieldDataType.DefaultTypes.LONG) {
-                return new TermsStatsLongFacetCollector(facetName, keyField, valueField, size, comparatorType, context, scriptLang, script, params);
-            } else if (keyFieldMapper.fieldDataType() == FieldDataType.DefaultTypes.INT) {
-                return new TermsStatsLongFacetCollector(facetName, keyField, valueField, size, comparatorType, context, scriptLang, script, params);
-            } else if (keyFieldMapper.fieldDataType() == FieldDataType.DefaultTypes.SHORT) {
-                return new TermsStatsLongFacetCollector(facetName, keyField, valueField, size, comparatorType, context, scriptLang, script, params);
-            } else if (keyFieldMapper.fieldDataType() == FieldDataType.DefaultTypes.BYTE) {
-                return new TermsStatsLongFacetCollector(facetName, keyField, valueField, size, comparatorType, context, scriptLang, script, params);
-            } else if (keyFieldMapper.fieldDataType() == FieldDataType.DefaultTypes.DOUBLE) {
-                return new TermsStatsDoubleFacetCollector(facetName, keyField, valueField, size, comparatorType, context, scriptLang, script, params);
-            } else if (keyFieldMapper.fieldDataType() == FieldDataType.DefaultTypes.FLOAT) {
-                return new TermsStatsDoubleFacetCollector(facetName, keyField, valueField, size, comparatorType, context, scriptLang, script, params);
-            }
+        FieldMapper keyMapper = context.smartNameFieldMapper(keyField);
+        if (keyMapper == null) {
+            throw new FacetPhaseExecutionException(facetName, "failed to find mapping for " + keyField);
+        }
+        IndexFieldData keyIndexFieldData = context.fieldData().getForField(keyMapper);
+
+        IndexNumericFieldData valueIndexFieldData = null;
+        SearchScript valueScript = null;
+        if (valueField != null) {
+            valueIndexFieldData = context.fieldData().getForField(context.smartNameFieldMapper(valueField));
+        } else {
+            valueScript = context.scriptService().search(context.lookup(), scriptLang, script, params);
         }
 
-        return new TermsStatsStringFacetCollector(facetName, keyField, valueField, size, comparatorType, context, scriptLang, script, params);
+        if (keyIndexFieldData instanceof IndexNumericFieldData) {
+            IndexNumericFieldData keyIndexNumericFieldData = (IndexNumericFieldData) keyIndexFieldData;
+            if (keyIndexNumericFieldData.getNumericType().isFloatingPoint()) {
+                return new TermsStatsDoubleFacetCollector(facetName, keyIndexNumericFieldData, valueIndexFieldData, valueScript, size, comparatorType, context);
+            } else {
+                return new TermsStatsLongFacetCollector(facetName, keyIndexNumericFieldData, valueIndexFieldData, valueScript, size, comparatorType, context);
+            }
+        }
+        return new TermsStatsStringFacetCollector(facetName, keyIndexFieldData, valueIndexFieldData, valueScript, size, comparatorType, context);
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java
index 637233a57da..248b2eec0ae 100644
--- a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/TermsStatsDoubleFacetCollector.java
@@ -23,13 +23,10 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.trove.ExtTDoubleObjectHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
@@ -40,67 +37,35 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
 
 public class TermsStatsDoubleFacetCollector extends AbstractFacetCollector {
 
     private final TermsStatsFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final String keyFieldName;
-
-    private final String valueFieldName;
+    private final IndexNumericFieldData keyIndexFieldData;
+    private final IndexNumericFieldData valueIndexFieldData;
+    private final SearchScript script;
 
     private final int size;
-
     private final int numberOfShards;
 
-    private final FieldDataType keyFieldDataType;
-
-    private NumericFieldData keyFieldData;
-
-    private final FieldDataType valueFieldDataType;
-
-    private final SearchScript script;
-
     private final Aggregator aggregator;
 
-    public TermsStatsDoubleFacetCollector(String facetName, String keyFieldName, String valueFieldName, int size, TermsStatsFacet.ComparatorType comparatorType,
-                                          SearchContext context, String scriptLang, String script, Map<String, Object> params) {
+    private DoubleValues keyValues;
+
+    public TermsStatsDoubleFacetCollector(String facetName, IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, SearchScript script,
+                                          int size, TermsStatsFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(keyFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            this.keyFieldName = keyFieldName;
-            this.keyFieldDataType = FieldDataType.DefaultTypes.STRING;
-        } else {
-            // add type filter if there is exact doc mapper associated with it
-            if (smartMappers.explicitTypeInNameWithDocMapper()) {
-                setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-            }
-
-            this.keyFieldName = smartMappers.mapper().names().indexName();
-            this.keyFieldDataType = smartMappers.mapper().fieldDataType();
-        }
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueIndexFieldData = valueIndexFieldData;
+        this.script = script;
 
         if (script == null) {
-            smartMappers = context.smartFieldMappers(valueFieldName);
-            if (smartMappers == null || !smartMappers.hasMapper()) {
-                throw new ElasticSearchIllegalArgumentException("failed to find mappings for [" + valueFieldName + "]");
-            }
-            this.valueFieldName = smartMappers.mapper().names().indexName();
-            this.valueFieldDataType = smartMappers.mapper().fieldDataType();
-            this.script = null;
             this.aggregator = new Aggregator();
         } else {
-            this.valueFieldName = null;
-            this.valueFieldDataType = null;
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
             this.aggregator = new ScriptAggregator(this.script);
         }
     }
@@ -114,17 +79,17 @@ public class TermsStatsDoubleFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        keyFieldData = (NumericFieldData) fieldDataCache.cache(keyFieldDataType, context.reader(), keyFieldName);
+        keyValues = keyIndexFieldData.load(context).getDoubleValues();
         if (script != null) {
             script.setNextReader(context);
         } else {
-            aggregator.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), valueFieldName);
+            aggregator.valueFieldData = valueIndexFieldData.load(context).getDoubleValues();
         }
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        keyFieldData.forEachValueInDoc(doc, aggregator);
+        keyValues.forEachValueInDoc(doc, aggregator);
     }
 
     @Override
@@ -153,12 +118,12 @@ public class TermsStatsDoubleFacetCollector extends AbstractFacetCollector {
         return new InternalTermsStatsDoubleFacet(facetName, comparatorType, size, ordered, aggregator.missing);
     }
 
-    public static class Aggregator implements NumericFieldData.MissingDoubleValueInDocProc {
+    public static class Aggregator implements DoubleValues.ValueInDocProc {
 
         final ExtTDoubleObjectHashMap<InternalTermsStatsDoubleFacet.DoubleEntry> entries = CacheRecycler.popDoubleObjectMap();
         int missing;
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueFieldData;
 
         final ValueAggregator valueAggregator = new ValueAggregator();
 
@@ -179,10 +144,14 @@ public class TermsStatsDoubleFacetCollector extends AbstractFacetCollector {
             missing++;
         }
 
-        public static class ValueAggregator implements NumericFieldData.DoubleValueInDocProc {
+        public static class ValueAggregator implements DoubleValues.ValueInDocProc {
 
             InternalTermsStatsDoubleFacet.DoubleEntry doubleEntry;
 
+            @Override
+            public void onMissing(int docId) {
+            }
+
             @Override
             public void onValue(int docId, double value) {
                 if (value < doubleEntry.min) {
diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java
index c36a89a3441..2dfab251d1b 100644
--- a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/TermsStatsLongFacetCollector.java
@@ -23,13 +23,11 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
@@ -40,68 +38,35 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
 
 public class TermsStatsLongFacetCollector extends AbstractFacetCollector {
 
     private final TermsStatsFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final String keyFieldName;
-
-    private final String valueFieldName;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final FieldDataType keyFieldDataType;
-
-    private NumericFieldData keyFieldData;
-
-    private final FieldDataType valueFieldDataType;
-
+    private final IndexNumericFieldData keyIndexFieldData;
+    private final IndexNumericFieldData valueIndexFieldData;
     private final SearchScript script;
 
+    private final int size;
+    private final int numberOfShards;
 
     private final Aggregator aggregator;
 
-    public TermsStatsLongFacetCollector(String facetName, String keyFieldName, String valueFieldName, int size, TermsStatsFacet.ComparatorType comparatorType,
-                                        SearchContext context, String scriptLang, String script, Map<String, Object> params) {
+    private LongValues keyValues;
+
+    public TermsStatsLongFacetCollector(String facetName, IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, SearchScript script,
+                                        int size, TermsStatsFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(keyFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            this.keyFieldName = keyFieldName;
-            this.keyFieldDataType = FieldDataType.DefaultTypes.STRING;
-        } else {
-            // add type filter if there is exact doc mapper associated with it
-            if (smartMappers.explicitTypeInNameWithDocMapper()) {
-                setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-            }
-
-            this.keyFieldName = smartMappers.mapper().names().indexName();
-            this.keyFieldDataType = smartMappers.mapper().fieldDataType();
-        }
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueIndexFieldData = valueIndexFieldData;
+        this.script = script;
 
         if (script == null) {
-            smartMappers = context.smartFieldMappers(valueFieldName);
-            if (smartMappers == null || !smartMappers.hasMapper()) {
-                throw new ElasticSearchIllegalArgumentException("failed to find mappings for [" + valueFieldName + "]");
-            }
-            this.valueFieldName = smartMappers.mapper().names().indexName();
-            this.valueFieldDataType = smartMappers.mapper().fieldDataType();
-            this.script = null;
             this.aggregator = new Aggregator();
         } else {
-            this.valueFieldName = null;
-            this.valueFieldDataType = null;
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
             this.aggregator = new ScriptAggregator(this.script);
         }
     }
@@ -115,17 +80,17 @@ public class TermsStatsLongFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        keyFieldData = (NumericFieldData) fieldDataCache.cache(keyFieldDataType, context.reader(), keyFieldName);
+        keyValues = keyIndexFieldData.load(context).getLongValues();
         if (script != null) {
             script.setNextReader(context);
         } else {
-            aggregator.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), valueFieldName);
+            aggregator.valueValues = valueIndexFieldData.load(context).getDoubleValues();
         }
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        keyFieldData.forEachValueInDoc(doc, aggregator);
+        keyValues.forEachValueInDoc(doc, aggregator);
     }
 
     @Override
@@ -155,13 +120,13 @@ public class TermsStatsLongFacetCollector extends AbstractFacetCollector {
         return new InternalTermsStatsLongFacet(facetName, comparatorType, size, ordered, aggregator.missing);
     }
 
-    public static class Aggregator implements NumericFieldData.MissingLongValueInDocProc {
+    public static class Aggregator implements LongValues.ValueInDocProc {
 
         final ExtTLongObjectHashMap<InternalTermsStatsLongFacet.LongEntry> entries = CacheRecycler.popLongObjectMap();
 
         int missing;
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueValues;
 
         final ValueAggregator valueAggregator = new ValueAggregator();
 
@@ -174,7 +139,7 @@ public class TermsStatsLongFacetCollector extends AbstractFacetCollector {
             }
             longEntry.count++;
             valueAggregator.longEntry = longEntry;
-            valueFieldData.forEachValueInDoc(docId, valueAggregator);
+            valueValues.forEachValueInDoc(docId, valueAggregator);
         }
 
         @Override
@@ -182,10 +147,14 @@ public class TermsStatsLongFacetCollector extends AbstractFacetCollector {
             missing++;
         }
 
-        public static class ValueAggregator implements NumericFieldData.DoubleValueInDocProc {
+        public static class ValueAggregator implements DoubleValues.ValueInDocProc {
 
             InternalTermsStatsLongFacet.LongEntry longEntry;
 
+            @Override
+            public void onMissing(int docId) {
+            }
+
             @Override
             public void onValue(int docId, double value) {
                 if (value < longEntry.min) {
diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java
index cc0836b4e74..b7375c3a300 100644
--- a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java
+++ b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java
@@ -20,12 +20,11 @@
 package org.elasticsearch.search.facet.termsstats.strings;
 
 import com.google.common.collect.ImmutableList;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.HashedBytesRef;
 import org.elasticsearch.common.text.BytesText;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.trove.ExtTHashMap;
@@ -69,8 +68,8 @@ public class InternalTermsStatsStringFacet extends InternalTermsStatsFacet {
         double min;
         double max;
 
-        public StringEntry(BytesRef term, long count, long totalCount, double total, double min, double max) {
-            this(new BytesText(new BytesArray(term)), count, totalCount, total, min, max);
+        public StringEntry(HashedBytesRef term, long count, long totalCount, double total, double min, double max) {
+            this(new BytesText(new BytesArray(term.bytes)), count, totalCount, total, min, max);
         }
 
         public StringEntry(Text term, long count, long totalCount, double total, double min, double max) {
diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java
index bdff3fb3714..a02b9d41dfa 100644
--- a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java
@@ -23,15 +23,13 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.CacheRecycler;
+import org.elasticsearch.common.lucene.HashedBytesRef;
 import org.elasticsearch.common.trove.ExtTHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.HashedBytesValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
@@ -42,68 +40,36 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
 
 public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
 
     private final TermsStatsFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final String keyFieldName;
-
-    private final String valueFieldName;
+    private final IndexFieldData keyIndexFieldData;
+    private final IndexNumericFieldData valueIndexFieldData;
+    private final SearchScript script;
 
     private final int size;
-
     private final int numberOfShards;
 
-    private final FieldDataType keyFieldDataType;
-
-    private FieldData keyFieldData;
-
-    private final FieldDataType valueFieldDataType;
-
-    private final SearchScript script;
-
     private final Aggregator aggregator;
 
-    public TermsStatsStringFacetCollector(String facetName, String keyFieldName, String valueFieldName, int size, TermsStatsFacet.ComparatorType comparatorType,
-                                          SearchContext context, String scriptLang, String script, Map<String, Object> params) {
+    private HashedBytesValues keyValues;
+
+    public TermsStatsStringFacetCollector(String facetName, IndexFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, SearchScript valueScript,
+                                          int size, TermsStatsFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueIndexFieldData = valueIndexFieldData;
+        this.script = valueScript;
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
 
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(keyFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            this.keyFieldName = keyFieldName;
-            this.keyFieldDataType = FieldDataType.DefaultTypes.STRING;
+        if (script != null) {
+            this.aggregator = new ScriptAggregator(valueScript);
         } else {
-            // add type filter if there is exact doc mapper associated with it
-            if (smartMappers.explicitTypeInNameWithDocMapper()) {
-                setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-            }
-
-            this.keyFieldName = smartMappers.mapper().names().indexName();
-            this.keyFieldDataType = smartMappers.mapper().fieldDataType();
-        }
-
-        if (script == null) {
-            smartMappers = context.smartFieldMappers(valueFieldName);
-            if (smartMappers == null || !smartMappers.hasMapper()) {
-                throw new ElasticSearchIllegalArgumentException("failed to find mappings for [" + valueFieldName + "]");
-            }
-            this.valueFieldName = smartMappers.mapper().names().indexName();
-            this.valueFieldDataType = smartMappers.mapper().fieldDataType();
-            this.script = null;
             this.aggregator = new Aggregator();
-        } else {
-            this.valueFieldName = null;
-            this.valueFieldDataType = null;
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-            this.aggregator = new ScriptAggregator(this.script);
         }
     }
 
@@ -116,17 +82,17 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        keyFieldData = fieldDataCache.cache(keyFieldDataType, context.reader(), keyFieldName);
+        keyValues = keyIndexFieldData.load(context).getHashedBytesValues();
         if (script != null) {
             script.setNextReader(context);
         } else {
-            aggregator.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), valueFieldName);
+            aggregator.valueValues = valueIndexFieldData.load(context).getDoubleValues();
         }
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        keyFieldData.forEachValueInDoc(doc, aggregator);
+        keyValues.forEachValueInDoc(doc, aggregator);
     }
 
     @Override
@@ -155,27 +121,28 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
         return new InternalTermsStatsStringFacet(facetName, comparatorType, size, ordered, aggregator.missing);
     }
 
-    public static class Aggregator implements FieldData.StringValueInDocProc {
+    public static class Aggregator implements HashedBytesValues.ValueInDocProc {
 
-        // LUCENE 4 UPGRADE: check if hashcode is not too expensive
-        final ExtTHashMap<BytesRef, InternalTermsStatsStringFacet.StringEntry> entries = CacheRecycler.popHashMap();
+        final ExtTHashMap<HashedBytesRef, InternalTermsStatsStringFacet.StringEntry> entries = CacheRecycler.popHashMap();
 
         int missing = 0;
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueValues;
 
         ValueAggregator valueAggregator = new ValueAggregator();
 
         @Override
-        public void onValue(int docId, BytesRef value) {
+        public void onValue(int docId, HashedBytesRef value) {
             InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(value);
             if (stringEntry == null) {
+                // we use "unsafe" hashedBytes, and only copy over if we "miss" on the map, and need to put it there
+                value = value.deepCopy();
                 stringEntry = new InternalTermsStatsStringFacet.StringEntry(value, 0, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
                 entries.put(value, stringEntry);
             }
             stringEntry.count++;
             valueAggregator.stringEntry = stringEntry;
-            valueFieldData.forEachValueInDoc(docId, valueAggregator);
+            valueValues.forEachValueInDoc(docId, valueAggregator);
         }
 
         @Override
@@ -183,10 +150,14 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
             missing++;
         }
 
-        public static class ValueAggregator implements NumericFieldData.DoubleValueInDocProc {
+        public static class ValueAggregator implements DoubleValues.ValueInDocProc {
 
             InternalTermsStatsStringFacet.StringEntry stringEntry;
 
+            @Override
+            public void onMissing(int docId) {
+            }
+
             @Override
             public void onValue(int docId, double value) {
                 if (value < stringEntry.min) {
@@ -209,9 +180,11 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
         }
 
         @Override
-        public void onValue(int docId, BytesRef value) {
+        public void onValue(int docId, HashedBytesRef value) {
             InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(value);
             if (stringEntry == null) {
+                // we use "unsafe" hashedBytes, and only copy over if we "miss" on the map, and need to put it there
+                value = value.deepCopy();
                 stringEntry = new InternalTermsStatsStringFacet.StringEntry(value, 1, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
                 entries.put(value, stringEntry);
             } else {

From 1765b0b813ea596008a913e09b6cc6e30d23f886 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 12:17:22 +0100
Subject: [PATCH 12/35] date histogram to use new field data

---
 .../CountDateHistogramFacetCollector.java     | 48 ++++---------
 .../DateHistogramFacetProcessor.java          | 33 +++++----
 .../ValueDateHistogramFacetCollector.java     | 70 +++++++------------
 ...alueScriptDateHistogramFacetCollector.java | 52 ++++----------
 4 files changed, 71 insertions(+), 132 deletions(-)

diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java
index 69fb71f5eb3..217bb795f3c 100644
--- a/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetCollector.java
@@ -23,14 +23,10 @@ import gnu.trove.map.hash.TLongLongHashMap;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.joda.TimeZoneRounding;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -41,48 +37,28 @@ import java.io.IOException;
  */
 public class CountDateHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final DateHistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private LongFieldData fieldData;
-
+    private LongValues values;
     private final DateHistogramProc histoProc;
 
-    public CountDateHistogramFacetCollector(String facetName, String fieldName, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public CountDateHistogramFacetCollector(String facetName, IndexNumericFieldData indexFieldData, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        FieldMapper mapper = smartMappers.mapper();
-
-        indexFieldName = mapper.names().indexName();
-        fieldDataType = mapper.fieldDataType();
-        histoProc = new DateHistogramProc(tzRounding);
+        this.indexFieldData = indexFieldData;
+        this.histoProc = new DateHistogramProc(tzRounding);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, histoProc);
+        values.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getLongValues();
     }
 
     @Override
@@ -90,7 +66,7 @@ public class CountDateHistogramFacetCollector extends AbstractFacetCollector {
         return new InternalCountDateHistogramFacet(facetName, comparatorType, histoProc.counts(), true);
     }
 
-    public static class DateHistogramProc implements LongFieldData.LongValueInDocProc {
+    public static class DateHistogramProc implements LongValues.ValueInDocProc {
 
         private final TLongLongHashMap counts = CacheRecycler.popLongLongMap();
 
@@ -100,6 +76,10 @@ public class CountDateHistogramFacetCollector extends AbstractFacetCollector {
             this.tzRounding = tzRounding;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, long value) {
             counts.adjustOrPutValue(tzRounding.calc(value), 1, 1);
diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java
index b4e2c237a29..ce8ca02accc 100644
--- a/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java
@@ -28,8 +28,9 @@ import org.elasticsearch.common.joda.TimeZoneRounding;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.FacetCollector;
 import org.elasticsearch.search.facet.FacetPhaseExecutionException;
@@ -138,21 +139,19 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa
             }
         }
 
+        if (interval == null) {
+            throw new FacetPhaseExecutionException(facetName, "[interval] is required to be set for histogram facet");
+        }
+
         if (keyField == null) {
             throw new FacetPhaseExecutionException(facetName, "key field is required to be set for histogram facet, either using [field] or using [key_field]");
         }
 
-        FieldMapper mapper = context.smartNameFieldMapper(keyField);
-        if (mapper == null) {
+        FieldMapper keyMapper = context.smartNameFieldMapper(keyField);
+        if (keyMapper == null) {
             throw new FacetPhaseExecutionException(facetName, "(key) field [" + keyField + "] not found");
         }
-        if (mapper.fieldDataType() != FieldDataType.DefaultTypes.LONG) {
-            throw new FacetPhaseExecutionException(facetName, "(key) field [" + keyField + "] is not of type date");
-        }
-
-        if (interval == null) {
-            throw new FacetPhaseExecutionException(facetName, "[interval] is required to be set for histogram facet");
-        }
+        IndexNumericFieldData keyIndexFieldData = context.fieldData().getForField(keyMapper);
 
         TimeZoneRounding.Builder tzRoundingBuilder;
         DateFieldParser fieldParser = dateFieldParsers.get(interval);
@@ -171,11 +170,17 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa
                 .build();
 
         if (valueScript != null) {
-            return new ValueScriptDateHistogramFacetCollector(facetName, keyField, scriptLang, valueScript, params, tzRounding, comparatorType, context);
-        } else if (valueField == null) {
-            return new CountDateHistogramFacetCollector(facetName, keyField, tzRounding, comparatorType, context);
+            SearchScript script = context.scriptService().search(context.lookup(), scriptLang, valueScript, params);
+            return new ValueScriptDateHistogramFacetCollector(facetName, keyIndexFieldData, script, tzRounding, comparatorType, context);
+        } else if (valueField != null) {
+            FieldMapper valueMapper = context.smartNameFieldMapper(valueField);
+            if (valueMapper == null) {
+                throw new FacetPhaseExecutionException(facetName, "(value) field [" + valueField + "] not found");
+            }
+            IndexNumericFieldData valueIndexFieldData = context.fieldData().getForField(valueMapper);
+            return new ValueDateHistogramFacetCollector(facetName, keyIndexFieldData, valueIndexFieldData, tzRounding, comparatorType, context);
         } else {
-            return new ValueDateHistogramFacetCollector(facetName, keyField, valueField, tzRounding, comparatorType, context);
+            return new CountDateHistogramFacetCollector(facetName, keyIndexFieldData, tzRounding, comparatorType, context);
         }
     }
 
diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java
index 4cba87d96fa..281d05712eb 100644
--- a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetCollector.java
@@ -23,15 +23,11 @@ import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.joda.TimeZoneRounding;
 import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -41,57 +37,31 @@ import java.io.IOException;
  */
 public class ValueDateHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String keyIndexFieldName;
-    private final String valueIndexFieldName;
+    private final IndexNumericFieldData keyIndexFieldData;
+    private final IndexNumericFieldData valueIndexFieldData;
 
     private final DateHistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType keyFieldDataType;
-    private LongFieldData keyFieldData;
-
-    private final FieldDataType valueFieldDataType;
-
+    private LongValues keyValues;
     private final DateHistogramProc histoProc;
 
-    public ValueDateHistogramFacetCollector(String facetName, String keyFieldName, String valueFieldName, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public ValueDateHistogramFacetCollector(String facetName, IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(keyFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + keyFieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        keyIndexFieldName = smartMappers.mapper().names().indexName();
-        keyFieldDataType = smartMappers.mapper().fieldDataType();
-
-        FieldMapper mapper = context.smartNameFieldMapper(valueFieldName);
-        if (mapper == null) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for value_field [" + valueFieldName + "]");
-        }
-        valueIndexFieldName = mapper.names().indexName();
-        valueFieldDataType = mapper.fieldDataType();
-
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueIndexFieldData = valueIndexFieldData;
         this.histoProc = new DateHistogramProc(tzRounding);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        keyFieldData.forEachValueInDoc(doc, histoProc);
+        keyValues.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        keyFieldData = (LongFieldData) fieldDataCache.cache(keyFieldDataType, context.reader(), keyIndexFieldName);
-        histoProc.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), valueIndexFieldName);
+        keyValues = keyIndexFieldData.load(context).getLongValues();
+        histoProc.valueValues = valueIndexFieldData.load(context).getDoubleValues();
     }
 
     @Override
@@ -99,13 +69,13 @@ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector {
         return new InternalFullDateHistogramFacet(facetName, comparatorType, histoProc.entries, true);
     }
 
-    public static class DateHistogramProc implements LongFieldData.LongValueInDocProc {
+    public static class DateHistogramProc implements LongValues.ValueInDocProc {
 
         final ExtTLongObjectHashMap<InternalFullDateHistogramFacet.FullEntry> entries = CacheRecycler.popLongObjectMap();
 
         private final TimeZoneRounding tzRounding;
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueValues;
 
         final ValueAggregator valueAggregator = new ValueAggregator();
 
@@ -113,6 +83,10 @@ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector {
             this.tzRounding = tzRounding;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, long value) {
             long time = tzRounding.calc(value);
@@ -124,13 +98,17 @@ public class ValueDateHistogramFacetCollector extends AbstractFacetCollector {
             }
             entry.count++;
             valueAggregator.entry = entry;
-            valueFieldData.forEachValueInDoc(docId, valueAggregator);
+            valueValues.forEachValueInDoc(docId, valueAggregator);
         }
 
-        public static class ValueAggregator implements NumericFieldData.DoubleValueInDocProc {
+        public static class ValueAggregator implements DoubleValues.ValueInDocProc {
 
             InternalFullDateHistogramFacet.FullEntry entry;
 
+            @Override
+            public void onMissing(int docId) {
+            }
+
             @Override
             public void onValue(int docId, double value) {
                 entry.totalCount++;
diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java
index bbb0d481308..e340e102669 100644
--- a/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java
@@ -24,19 +24,14 @@ import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.joda.TimeZoneRounding;
 import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Map;
 
 /**
  * A histogram facet collector that uses the same field as the key as well as the
@@ -44,48 +39,25 @@ import java.util.Map;
  */
 public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollector {
 
-    private final String indexFieldName;
+    private final IndexNumericFieldData keyIndexFieldData;
 
     private final DateHistogramFacet.ComparatorType comparatorType;
 
-    private final FieldDataCache fieldDataCache;
-
-    private final FieldDataType fieldDataType;
-
-    private LongFieldData fieldData;
-
+    private LongValues keyValues;
     private final SearchScript valueScript;
-
     private final DateHistogramProc histoProc;
 
-    public ValueScriptDateHistogramFacetCollector(String facetName, String fieldName, String scriptLang, String valueScript, Map<String, Object> params, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) {
+    public ValueScriptDateHistogramFacetCollector(String facetName, IndexNumericFieldData keyIndexFieldData, SearchScript valueScript, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, SearchContext context) {
         super(facetName);
         this.comparatorType = comparatorType;
-        this.fieldDataCache = context.fieldDataCache();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        this.valueScript = context.scriptService().search(context.lookup(), scriptLang, valueScript, params);
-
-        FieldMapper mapper = smartMappers.mapper();
-
-        indexFieldName = mapper.names().indexName();
-        fieldDataType = mapper.fieldDataType();
-
+        this.keyIndexFieldData = keyIndexFieldData;
+        this.valueScript = valueScript;
         histoProc = new DateHistogramProc(tzRounding, this.valueScript);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, histoProc);
+        keyValues.forEachValueInDoc(doc, histoProc);
     }
 
     @Override
@@ -95,7 +67,7 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        keyValues = keyIndexFieldData.load(context).getLongValues();
         valueScript.setNextReader(context);
     }
 
@@ -104,7 +76,7 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect
         return new InternalFullDateHistogramFacet(facetName, comparatorType, histoProc.entries, true);
     }
 
-    public static class DateHistogramProc implements LongFieldData.LongValueInDocProc {
+    public static class DateHistogramProc implements LongValues.ValueInDocProc {
 
         private final TimeZoneRounding tzRounding;
 
@@ -117,6 +89,10 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect
             this.valueScript = valueScript;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, long value) {
             valueScript.setNextDocId(docId);

From d88e3f73acf0d90babefd1c05d72b0a4ce424dc4 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 15:01:31 +0100
Subject: [PATCH 13/35] add specific makeSafe method to make an unsafe (shared)
 bytes based value to a "safe" one

---
 .../index/fielddata/BytesValues.java          | 19 +++++++++++++++----
 .../index/fielddata/HashedBytesValues.java    | 16 ++++++++++++++++
 .../ConcreteBytesRefAtomicFieldData.java      | 13 +++++++++++++
 .../TermsStatsStringFacetCollector.java       |  8 ++++----
 4 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
index 9b4ade8ea61..20d1502d632 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
@@ -28,15 +28,20 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
  */
 public interface BytesValues {
 
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
     /**
      * Is there a value for this doc?
      */
     boolean hasValue(int docId);
 
     /**
-     * Is one of the documents in this field data values is multi valued?
+     * Converts the provided bytes to "safe" ones from a "non" safe call made (if needed).
      */
-    boolean isMultiValued();
+    BytesRef makeSafe(BytesRef bytes);
 
     /**
      * Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
@@ -147,14 +152,20 @@ public interface BytesValues {
             this.values = values;
         }
 
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
         @Override
         public boolean hasValue(int docId) {
             return values.hasValue(docId);
         }
 
         @Override
-        public boolean isMultiValued() {
-            return values.isMultiValued();
+        public BytesRef makeSafe(BytesRef bytes) {
+            // we need to make a copy, since we use scratch to provide it
+            return BytesRef.deepCopyOf(bytes);
         }
 
         @Override
diff --git a/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
index 0658a4797a1..2fe6f6da61b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
@@ -37,6 +37,11 @@ public interface HashedBytesValues {
      */
     boolean hasValue(int docId);
 
+    /**
+     * Converts the provided bytes to "safe" ones from a "non" safe call made (if needed).
+     */
+    HashedBytesRef makeSafe(HashedBytesRef bytes);
+
     /**
      * Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
      */
@@ -146,6 +151,11 @@ public interface HashedBytesValues {
             return values.hasValue(docId);
         }
 
+        @Override
+        public HashedBytesRef makeSafe(HashedBytesRef bytes) {
+            return new HashedBytesRef(values.makeSafe(bytes.bytes), bytes.hash);
+        }
+
         @Override
         public HashedBytesRef getValue(int docId) {
             BytesRef value = values.getValue(docId);
@@ -289,6 +299,12 @@ public interface HashedBytesValues {
             return values.hasValue(docId);
         }
 
+        @Override
+        public HashedBytesRef makeSafe(HashedBytesRef bytes) {
+            // we use scratch to provide it, so just need to copy it over to a new instance
+            return new HashedBytesRef(bytes.bytes, bytes.hash);
+        }
+
         @Override
         public HashedBytesRef getValue(int docId) {
             String value = values.getValue(docId);
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
index 55895305cec..ea036083744 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
@@ -146,6 +146,12 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
             return ordinals.getOrd(docId) != 0;
         }
 
+        @Override
+        public BytesRef makeSafe(BytesRef bytes) {
+            // no need to do anything, its already concrete bytes...
+            return bytes;
+        }
+
         @Override
         public BytesRef getValue(int docId) {
             return values[ordinals.getOrd(docId)];
@@ -337,6 +343,13 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
             return ordinals.getOrd(docId) != 0;
         }
 
+        @Override
+        public HashedBytesRef makeSafe(HashedBytesRef bytes) {
+            // we just need to create a copy of the bytes ref, no need
+            // to create a copy of the actual BytesRef, as its concrete
+            return new HashedBytesRef(bytes.bytes, bytes.hash);
+        }
+
         @Override
         public HashedBytesRef getValue(int docId) {
             int ord = ordinals.getOrd(docId);
diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java
index a02b9d41dfa..e75d8c5bb3f 100644
--- a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetCollector.java
@@ -83,6 +83,7 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
         keyValues = keyIndexFieldData.load(context).getHashedBytesValues();
+        aggregator.keyValues = keyValues;
         if (script != null) {
             script.setNextReader(context);
         } else {
@@ -127,6 +128,7 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
 
         int missing = 0;
 
+        HashedBytesValues keyValues;
         DoubleValues valueValues;
 
         ValueAggregator valueAggregator = new ValueAggregator();
@@ -135,8 +137,7 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
         public void onValue(int docId, HashedBytesRef value) {
             InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(value);
             if (stringEntry == null) {
-                // we use "unsafe" hashedBytes, and only copy over if we "miss" on the map, and need to put it there
-                value = value.deepCopy();
+                value = keyValues.makeSafe(value);
                 stringEntry = new InternalTermsStatsStringFacet.StringEntry(value, 0, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
                 entries.put(value, stringEntry);
             }
@@ -183,8 +184,7 @@ public class TermsStatsStringFacetCollector extends AbstractFacetCollector {
         public void onValue(int docId, HashedBytesRef value) {
             InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(value);
             if (stringEntry == null) {
-                // we use "unsafe" hashedBytes, and only copy over if we "miss" on the map, and need to put it there
-                value = value.deepCopy();
+                value = keyValues.makeSafe(value);
                 stringEntry = new InternalTermsStatsStringFacet.StringEntry(value, 1, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
                 entries.put(value, stringEntry);
             } else {

From 2e86081f7b1dbda0e925be5cd4e6a9887e9a56ca Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 15:15:10 +0100
Subject: [PATCH 14/35] use smartNameMapper on context

---
 .../search/facet/statistical/StatisticalFacetProcessor.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java
index 62e9b84ed45..98470cb7297 100644
--- a/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/statistical/StatisticalFacetProcessor.java
@@ -91,7 +91,7 @@ public class StatisticalFacetProcessor extends AbstractComponent implements Face
         if (fieldsNames != null) {
             IndexNumericFieldData[] indexFieldDatas = new IndexNumericFieldData[fieldsNames.length];
             for (int i = 0; i < fieldsNames.length; i++) {
-                FieldMapper fieldMapper = context.mapperService().smartNameFieldMapper(fieldsNames[i]);
+                FieldMapper fieldMapper = context.smartNameFieldMapper(fieldsNames[i]);
                 if (fieldMapper == null) {
                     throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldsNames[i] + "]");
                 }
@@ -103,7 +103,7 @@ public class StatisticalFacetProcessor extends AbstractComponent implements Face
             throw new FacetPhaseExecutionException(facetName, "statistical facet requires either [script] or [field] to be set");
         }
         if (field != null) {
-            FieldMapper fieldMapper = context.mapperService().smartNameFieldMapper(field);
+            FieldMapper fieldMapper = context.smartNameFieldMapper(field);
             if (fieldMapper == null) {
                 throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + field + "]");
             }

From d82859c82ba5ad954419d174558e0e9b32a3852d Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 17:40:52 +0100
Subject: [PATCH 15/35] geo point new field mapper with geo distance facet
 based impl

---
 .../fielddata/AtomicGeoPointFieldData.java    |  27 +
 .../index/fielddata/GeoPointValues.java       | 122 +++
 .../fielddata/IndexFieldDataService.java      |   3 +
 .../fielddata/IndexGeoPointFieldData.java     |  37 +
 .../index/fielddata/ScriptDocValues.java      |  29 +
 .../GeoPointDoubleArrayAtomicFieldData.java   | 756 ++++++++++++++++++
 .../GeoPointDoubleArrayIndexFieldData.java    | 160 ++++
 .../fielddata/util/GeoPointArrayRef.java      | 164 ++++
 .../index/mapper/geo/GeoPoint.java            |  33 +-
 .../index/mapper/geo/GeoPointFieldMapper.java |   5 +
 .../GeoDistanceFacetCollector.java            |  47 +-
 .../GeoDistanceFacetProcessor.java            |  22 +-
 .../ScriptGeoDistanceFacetCollector.java      |  13 +-
 .../ValueGeoDistanceFacetCollector.java       |  45 +-
 14 files changed, 1397 insertions(+), 66 deletions(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/util/GeoPointArrayRef.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
new file mode 100644
index 00000000000..ddfae5883fd
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+/**
+ */
+public interface AtomicGeoPointFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
+
+    GeoPointValues getGeoPointValues();
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java b/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java
new file mode 100644
index 00000000000..83707e2a478
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.GeoPointArrayRef;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
+
+/**
+ */
+public interface GeoPointValues {
+
+    /**
+     * Is one of the documents in this field data values is multi valued?
+     */
+    boolean isMultiValued();
+
+    /**
+     * Is there a value for this doc?
+     */
+    boolean hasValue(int docId);
+
+    GeoPoint getValue(int docId);
+
+    GeoPoint getValueSafe(int docId);
+
+    GeoPointArrayRef getValues(int docId);
+
+    Iter getIter(int docId);
+
+    Iter getIterSafe(int docId);
+
+    /**
+     * Go over all the possible values in their geo point format for a specific doc.
+     */
+    void forEachValueInDoc(int docId, ValueInDocProc proc);
+
+    /**
+     * Go over all the possible values in their geo point format for a specific doc.
+     */
+    void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
+
+    public static interface ValueInDocProc {
+        void onValue(int docId, GeoPoint value);
+
+        void onMissing(int docId);
+    }
+
+    /**
+     * Go over all the possible values in their geo point format for a specific doc.
+     */
+    void forEachLatLonValueInDoc(int docId, LatLonValueInDocProc proc);
+
+    public static interface LatLonValueInDocProc {
+        void onValue(int docId, double lat, double lon);
+
+        void onMissing(int docId);
+    }
+
+    static interface Iter {
+
+        boolean hasNext();
+
+        GeoPoint next();
+
+        static class Empty implements Iter {
+
+            public static final Empty INSTANCE = new Empty();
+
+            @Override
+            public boolean hasNext() {
+                return false;
+            }
+
+            @Override
+            public GeoPoint next() {
+                throw new ElasticSearchIllegalStateException();
+            }
+        }
+
+        static class Single implements Iter {
+
+            public GeoPoint value;
+            public boolean done;
+
+            public Single reset(GeoPoint value) {
+                this.value = value;
+                this.done = false;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return !done;
+            }
+
+            @Override
+            public GeoPoint next() {
+                assert !done;
+                done = true;
+                return value;
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index c942059b0ce..ed6b277b588 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.plain.ConcreteBytesRefIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.GeoPointDoubleArrayIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.LongArrayIndexFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -49,12 +50,14 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put("string", new ConcreteBytesRefIndexFieldData.Builder())
                 .put("double", new DoubleArrayIndexFieldData.Builder())
                 .put("long", new LongArrayIndexFieldData.Builder())
+                .put("geo_point", new GeoPointDoubleArrayIndexFieldData.Builder())
                 .immutableMap();
 
         buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("long", "array"), new LongArrayIndexFieldData.Builder())
+                .put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder())
                 .immutableMap();
     }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java
new file mode 100644
index 00000000000..ccd8550ed5d
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.AtomicReaderContext;
+
+/**
+ */
+public interface IndexGeoPointFieldData<FD extends AtomicGeoPointFieldData> extends IndexFieldData<FD> {
+
+    /**
+     * Loads the atomic field data for the reader, possibly cached.
+     */
+    FD load(AtomicReaderContext context);
+
+    /**
+     * Loads directly the atomic field data for the reader, ignoring any caching involved.
+     */
+    FD loadDirect(AtomicReaderContext context) throws Exception;
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index bf60565b769..cdf334ce8d1 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.fielddata.util.*;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
 
 /**
  * Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
@@ -255,4 +256,32 @@ public interface ScriptDocValues {
             return values.getValues(docId);
         }
     }
+
+    static class GeoPoints implements ScriptDocValues {
+
+        private final GeoPointValues values;
+        private int docId;
+
+        public GeoPoints(GeoPointValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            this.docId = docId;
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return !values.hasValue(docId);
+        }
+
+        public GeoPoint getValue() {
+            return values.getValue(docId);
+        }
+
+        public GeoPointArrayRef getValues() {
+            return values.getValues(docId);
+        }
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
new file mode 100644
index 00000000000..be12d913ceb
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
@@ -0,0 +1,756 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.BytesValues;
+import org.elasticsearch.index.fielddata.HashedBytesValues;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.GeoPointArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
+import org.elasticsearch.index.search.geo.GeoHashUtils;
+
+/**
+ */
+public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPointFieldData {
+
+    protected final double[] lon;
+    protected final double[] lat;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public GeoPointDoubleArrayAtomicFieldData(double[] lon, double[] lat, int numDocs) {
+        this.lon = lon;
+        this.lat = lat;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    @Override
+    public ScriptDocValues getScriptValues() {
+        return new ScriptDocValues.GeoPoints(getGeoPointValues());
+    }
+
+    public static class WithOrdinals extends GeoPointDoubleArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(double[] lon, double[] lat, int numDocs, Ordinals ordinals) {
+            super(lon, lat, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + (RamUsage.NUM_BYTES_ARRAY_HEADER + (lon.length * RamUsage.NUM_BYTES_DOUBLE)) + (RamUsage.NUM_BYTES_ARRAY_HEADER + (lat.length * RamUsage.NUM_BYTES_DOUBLE)) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(lon, lat, ordinals.ordinals());
+        }
+
+        @Override
+        public GeoPointValues getGeoPointValues() {
+            return new GeoPointValues(lon, lat, ordinals.ordinals());
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] lon;
+            private final double[] lat;
+            private final Ordinals.Docs ordinals;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final ValuesIter valuesIter;
+
+            StringValues(double[] lon, double[] lat, Ordinals.Docs ordinals) {
+                this.lon = lon;
+                this.lat = lat;
+                this.ordinals = ordinals;
+                this.valuesIter = new ValuesIter(lon, lat);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public String getValue(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return null;
+                }
+                return GeoHashUtils.encode(lat[ord], lon[ord]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return StringArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    int ord = ords.values[i];
+                    arrayScratch.values[arrayScratch.end++] = GeoHashUtils.encode(lat[ord], lon[ord]);
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return valuesIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, GeoHashUtils.encode(lat[ord], lon[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final double[] lon;
+                private final double[] lat;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] lon, double[] lat) {
+                    this.lon = lon;
+                    this.lat = lat;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public String next() {
+                    String value = GeoHashUtils.encode(lat[ord], lon[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class GeoPointValues implements org.elasticsearch.index.fielddata.GeoPointValues {
+
+            private final double[] lon;
+            private final double[] lat;
+            private final Ordinals.Docs ordinals;
+
+            private final GeoPoint scratch = new GeoPoint();
+            private final GeoPointArrayRef arrayScratch = new GeoPointArrayRef(new GeoPoint[1], 1);
+            private final ValuesIter valuesIter;
+            private final SafeValuesIter safeValuesIter;
+
+            GeoPointValues(double[] lon, double[] lat, Ordinals.Docs ordinals) {
+                this.lon = lon;
+                this.lat = lat;
+                this.ordinals = ordinals;
+                this.valuesIter = new ValuesIter(lon, lat);
+                this.safeValuesIter = new SafeValuesIter(lon, lat);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public GeoPoint getValue(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return null;
+                }
+                return scratch.reset(lat[ord], lon[ord]);
+            }
+
+            @Override
+            public GeoPoint getValueSafe(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return null;
+                }
+                return new GeoPoint(lat[ord], lon[ord]);
+            }
+
+            @Override
+            public GeoPointArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return GeoPointArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    int ord = ords.values[i];
+                    arrayScratch.values[arrayScratch.end++].reset(lat[ord], lon[ord]);
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return valuesIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return safeValuesIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, scratch.reset(lat[ord], lon[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, new GeoPoint(lat[ord], lon[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            @Override
+            public void forEachLatLonValueInDoc(int docId, LatLonValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, lat[ord], lon[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final double[] lon;
+                private final double[] lat;
+                private final GeoPoint scratch = new GeoPoint();
+
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(double[] lon, double[] lat) {
+                    this.lon = lon;
+                    this.lat = lat;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public GeoPoint next() {
+                    scratch.reset(lat[ord], lon[ord]);
+                    ord = ordsIter.next();
+                    return scratch;
+                }
+            }
+
+            static class SafeValuesIter implements Iter {
+
+                private final double[] lon;
+                private final double[] lat;
+
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                SafeValuesIter(double[] lon, double[] lat) {
+                    this.lon = lon;
+                    this.lat = lat;
+                }
+
+                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public GeoPoint next() {
+                    GeoPoint value = new GeoPoint(lat[ord], lon[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    /**
+     * Assumes unset values are marked in bitset, and docId is used as the index to the value array.
+     */
+    public static class SingleFixedSet extends GeoPointDoubleArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(double[] lon, double[] lat, int numDocs, FixedBitSet set) {
+            super(lon, lat, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + (RamUsage.NUM_BYTES_ARRAY_HEADER + (lon.length * RamUsage.NUM_BYTES_DOUBLE)) + (RamUsage.NUM_BYTES_ARRAY_HEADER + (lat.length * RamUsage.NUM_BYTES_DOUBLE)) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(lon, lat, set);
+        }
+
+        @Override
+        public GeoPointValues getGeoPointValues() {
+            return new GeoPointValues(lon, lat, set);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] lon;
+            private final double[] lat;
+            private final FixedBitSet set;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(double[] lon, double[] lat, FixedBitSet set) {
+                this.lon = lon;
+                this.lat = lat;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public String getValue(int docId) {
+                if (set.get(docId)) {
+                    return GeoHashUtils.encode(lat[docId], lon[docId]);
+                } else {
+                    return null;
+                }
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = GeoHashUtils.encode(lat[docId], lon[docId]);
+                    return arrayScratch;
+                } else {
+                    return StringArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(GeoHashUtils.encode(lat[docId], lon[docId]));
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, GeoHashUtils.encode(lat[docId], lon[docId]));
+                } else {
+                    proc.onMissing(docId);
+                }
+            }
+        }
+
+        static class GeoPointValues implements org.elasticsearch.index.fielddata.GeoPointValues {
+
+            private final double[] lon;
+            private final double[] lat;
+            private final FixedBitSet set;
+
+            private final GeoPoint scratch = new GeoPoint();
+            private final GeoPointArrayRef arrayScratch = new GeoPointArrayRef(new GeoPoint[1]);
+            private final Iter.Single iter = new Iter.Single();
+
+            GeoPointValues(double[] lon, double[] lat, FixedBitSet set) {
+                this.lon = lon;
+                this.lat = lat;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public GeoPoint getValue(int docId) {
+                if (set.get(docId)) {
+                    return scratch.reset(lat[docId], lon[docId]);
+                } else {
+                    return null;
+                }
+            }
+
+            @Override
+            public GeoPoint getValueSafe(int docId) {
+                if (set.get(docId)) {
+                    return new GeoPoint(lat[docId], lon[docId]);
+                } else {
+                    return null;
+                }
+            }
+
+            @Override
+            public GeoPointArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0].reset(lat[docId], lon[docId]);
+                    return arrayScratch;
+                } else {
+                    return GeoPointArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(scratch.reset(lat[docId], lon[docId]));
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(new GeoPoint(lat[docId], lon[docId]));
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, scratch.reset(lat[docId], lon[docId]));
+                } else {
+                    proc.onMissing(docId);
+                }
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, new GeoPoint(lat[docId], lon[docId]));
+                } else {
+                    proc.onMissing(docId);
+                }
+            }
+
+            @Override
+            public void forEachLatLonValueInDoc(int docId, LatLonValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, lat[docId], lon[docId]);
+                } else {
+                    proc.onMissing(docId);
+                }
+            }
+        }
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends GeoPointDoubleArrayAtomicFieldData {
+
+        public Single(double[] lon, double[] lat, int numDocs) {
+            super(lon, lat, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + (RamUsage.NUM_BYTES_ARRAY_HEADER + (lon.length * RamUsage.NUM_BYTES_DOUBLE)) + (RamUsage.NUM_BYTES_ARRAY_HEADER + (lat.length * RamUsage.NUM_BYTES_DOUBLE));
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues(lon, lat);
+        }
+
+        @Override
+        public GeoPointValues getGeoPointValues() {
+            return new GeoPointValues(lon, lat);
+        }
+
+        static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
+
+            private final double[] lon;
+            private final double[] lat;
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            StringValues(double[] lon, double[] lat) {
+                this.lon = lon;
+                this.lat = lat;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public String getValue(int docId) {
+                return GeoHashUtils.encode(lat[docId], lon[docId]);
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                arrayScratch.values[0] = GeoHashUtils.encode(lat[docId], lon[docId]);
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(GeoHashUtils.encode(lat[docId], lon[docId]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, GeoHashUtils.encode(lat[docId], lon[docId]));
+            }
+        }
+
+        static class GeoPointValues implements org.elasticsearch.index.fielddata.GeoPointValues {
+
+            private final double[] lon;
+            private final double[] lat;
+
+            private final GeoPoint scratch = new GeoPoint();
+            private final GeoPointArrayRef arrayScratch = new GeoPointArrayRef(new GeoPoint[1]);
+            private final Iter.Single iter = new Iter.Single();
+
+            GeoPointValues(double[] lon, double[] lat) {
+                this.lon = lon;
+                this.lat = lat;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public GeoPoint getValue(int docId) {
+                return scratch.reset(lat[docId], lon[docId]);
+            }
+
+            @Override
+            public GeoPoint getValueSafe(int docId) {
+                return new GeoPoint(lat[docId], lon[docId]);
+            }
+
+            @Override
+            public GeoPointArrayRef getValues(int docId) {
+                arrayScratch.values[0].reset(lat[docId], lon[docId]);
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(scratch.reset(lat[docId], lon[docId]));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return iter.reset(new GeoPoint(lat[docId], lon[docId]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, scratch.reset(lat[docId], lon[docId]));
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, new GeoPoint(lat[docId], lon[docId]));
+            }
+
+            @Override
+            public void forEachLatLonValueInDoc(int docId, LatLonValueInDocProc proc) {
+                proc.onValue(docId, lat[docId], lon[docId]);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
new file mode 100644
index 00000000000..11a89ee5bb4
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TDoubleArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<GeoPointDoubleArrayAtomicFieldData> implements IndexGeoPointFieldData<GeoPointDoubleArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new GeoPointDoubleArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        }
+    }
+
+    public GeoPointDoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public GeoPointDoubleArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public GeoPointDoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldNames().indexName());
+        if (terms == null) {
+            return new GeoPointDoubleArrayAtomicFieldData.Single(new double[0], new double[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TDoubleArrayList lat = new TDoubleArrayList();
+        final TDoubleArrayList lon = new TDoubleArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        lat.add(0); // first "t" indicates null value
+        lon.add(0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+
+                String location = term.utf8ToString();
+                int comma = location.indexOf(',');
+                lat.add(Double.parseDouble(location.substring(0, comma)));
+                lon.add(Double.parseDouble(location.substring(comma + 1)));
+
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            double[] sLat = new double[reader.maxDoc()];
+            double[] sLon = new double[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sLat[i] = lat.get(nativeOrdinal);
+                    sLon[i] = lon.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat, reader.maxDoc());
+            } else {
+                return new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(lon.toArray(new double[lon.size()]), lat.toArray(new double[lat.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        throw new ElasticSearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/fielddata/util/GeoPointArrayRef.java b/src/main/java/org/elasticsearch/index/fielddata/util/GeoPointArrayRef.java
new file mode 100644
index 00000000000..d4859f2dd3d
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/util/GeoPointArrayRef.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.util;
+
+import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
+
+import java.util.AbstractList;
+import java.util.RandomAccess;
+
+/**
+ */
+public class GeoPointArrayRef extends AbstractList<GeoPoint> implements RandomAccess {
+
+    public static final GeoPointArrayRef EMPTY = new GeoPointArrayRef(new GeoPoint[0]);
+
+    public GeoPoint[] values;
+    public int start;
+    public int end;
+
+    public GeoPointArrayRef(GeoPoint[] values) {
+        this(values, 0, values.length);
+    }
+
+    public GeoPointArrayRef(GeoPoint[] values, int length) {
+        this(values, 0, length);
+    }
+
+    public GeoPointArrayRef(GeoPoint[] values, int start, int end) {
+        this.values = values;
+        this.start = start;
+        this.end = end;
+        for (int i = start; i < end; i++) {
+            this.values[i] = new GeoPoint();
+        }
+    }
+
+    public void reset(int newLength) {
+        assert start == 0; // NOTE: senseless if offset != 0
+        end = 0;
+        if (values.length < newLength) {
+            values = new GeoPoint[ArrayUtil.oversize(newLength, 32)];
+            for (int i = 0; i < values.length; i++) {
+                values[i] = new GeoPoint();
+            }
+        }
+    }
+
+    @Override
+    public int size() {
+        return end - start;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size() != 0;
+    }
+
+    @Override
+    public GeoPoint get(int index) {
+        assert index >= 0 && index < size();
+        return values[start + index];
+    }
+
+    @Override
+    public boolean contains(Object target) {
+        if (!(target instanceof GeoPoint)) {
+            return false;
+        }
+        for (int i = start; i < end; i++) {
+            if (values[i].equals((GeoPoint) target)) return true;
+        }
+        return false;
+    }
+
+    @Override
+    public int indexOf(Object target) {
+        if (!(target instanceof GeoPoint)) {
+            return -1;
+        }
+        GeoPoint geoPoint = (GeoPoint) target;
+        for (int i = start; i < end; i++) {
+            if (values[i].equals(geoPoint)) return (i - start);
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object target) {
+        if (!(target instanceof GeoPoint)) {
+            return -1;
+        }
+        GeoPoint geoPoint = (GeoPoint) target;
+        for (int i = end - 1; i >= start; i--) {
+            if (values[i].equals(target)) return (i - start);
+        }
+        return -1;
+    }
+
+    @Override
+    public GeoPoint set(int index, GeoPoint element) {
+        assert index >= 0 && index < size();
+        GeoPoint oldValue = values[start + index];
+        values[start + index] = element;
+        return oldValue;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object instanceof GeoPointArrayRef) {
+            GeoPointArrayRef that = (GeoPointArrayRef) object;
+            int size = size();
+            if (that.size() != size) {
+                return false;
+            }
+            for (int i = 0; i < size; i++) {
+                if (!values[start + i].equals(that.values[that.start + i])) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return super.equals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = 1;
+        for (int i = start; i < end; i++) {
+            result = 31 * result + values[i].hashCode();
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(size() * 10);
+        builder.append('[').append(values[start]);
+        for (int i = start + 1; i < end; i++) {
+            builder.append(", ").append(values[i]);
+        }
+        return builder.append(']').toString();
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPoint.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPoint.java
index ae991d93374..50918588546 100644
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPoint.java
+++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPoint.java
@@ -27,10 +27,9 @@ import org.elasticsearch.index.search.geo.GeoHashUtils;
 public class GeoPoint {
 
     private double lat;
-
     private double lon;
 
-    GeoPoint() {
+    public GeoPoint() {
     }
 
     public GeoPoint(double lat, double lon) {
@@ -38,6 +37,12 @@ public class GeoPoint {
         this.lon = lon;
     }
 
+    public GeoPoint reset(double lat, double lon) {
+        this.lat = lat;
+        this.lon = lon;
+        return this;
+    }
+
     void latlon(double lat, double lon) {
         this.lat = lat;
         this.lon = lon;
@@ -66,4 +71,28 @@ public class GeoPoint {
     public final String getGeohash() {
         return GeoHashUtils.encode(lat, lon);
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        GeoPoint geoPoint = (GeoPoint) o;
+
+        if (Double.compare(geoPoint.lat, lat) != 0) return false;
+        if (Double.compare(geoPoint.lon, lon) != 0) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result;
+        long temp;
+        temp = lat != +0.0d ? Double.doubleToLongBits(lat) : 0L;
+        result = (int) (temp ^ (temp >>> 32));
+        temp = lon != +0.0d ? Double.doubleToLongBits(lon) : 0L;
+        result = 31 * result + (int) (temp ^ (temp >>> 32));
+        return result;
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
index 76918148e28..72d112e9dff 100644
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
@@ -580,6 +580,11 @@ public class GeoPointFieldMapper implements Mapper, ArrayValueMapperParser {
             return GeoPointFieldMapper.Defaults.FIELD_TYPE;
         }
 
+        @Override
+        public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+            return new org.elasticsearch.index.fielddata.FieldDataType("geo_point");
+        }
+
         @Override
         public FieldDataType fieldDataType() {
             return GeoPointFieldDataType.TYPE;
diff --git a/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetCollector.java
index 4144f8d1e07..21809970c9f 100644
--- a/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetCollector.java
@@ -21,14 +21,11 @@ package org.elasticsearch.search.facet.geodistance;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.search.geo.GeoDistance;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -38,10 +35,9 @@ import java.io.IOException;
  */
 public class GeoDistanceFacetCollector extends AbstractFacetCollector {
 
-    protected final String indexFieldName;
+    protected final IndexGeoPointFieldData indexFieldData;
 
     protected final double lat;
-
     protected final double lon;
 
     protected final DistanceUnit unit;
@@ -49,15 +45,12 @@ public class GeoDistanceFacetCollector extends AbstractFacetCollector {
     protected final GeoDistance geoDistance;
     protected final GeoDistance.FixedSourceDistance fixedSourceDistance;
 
-    protected final FieldDataCache fieldDataCache;
-
-    protected GeoPointFieldData fieldData;
+    protected GeoPointValues values;
 
     protected final GeoDistanceFacet.Entry[] entries;
+    protected GeoPointValues.LatLonValueInDocProc aggregator;
 
-    protected GeoPointFieldData.ValueInDocProc aggregator;
-
-    public GeoDistanceFacetCollector(String facetName, String fieldName, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
+    public GeoDistanceFacetCollector(String facetName, IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
                                      GeoDistanceFacet.Entry[] entries, SearchContext context) {
         super(facetName);
         this.lat = lat;
@@ -65,30 +58,14 @@ public class GeoDistanceFacetCollector extends AbstractFacetCollector {
         this.unit = unit;
         this.entries = entries;
         this.geoDistance = geoDistance;
-        this.fieldDataCache = context.fieldDataCache();
-
+        this.indexFieldData = indexFieldData;
         this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, unit);
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + fieldName + "]");
-        }
-        if (smartMappers.mapper().fieldDataType() != GeoPointFieldDataType.TYPE) {
-            throw new FacetPhaseExecutionException(facetName, "field [" + fieldName + "] is not a geo_point field");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
         this.aggregator = new Aggregator(fixedSourceDistance, entries);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getGeoPointValues();
     }
 
     @Override
@@ -96,7 +73,7 @@ public class GeoDistanceFacetCollector extends AbstractFacetCollector {
         for (GeoDistanceFacet.Entry entry : entries) {
             entry.foundInDoc = false;
         }
-        fieldData.forEachValueInDoc(doc, aggregator);
+        values.forEachLatLonValueInDoc(doc, aggregator);
     }
 
     @Override
@@ -104,7 +81,7 @@ public class GeoDistanceFacetCollector extends AbstractFacetCollector {
         return new InternalGeoDistanceFacet(facetName, entries);
     }
 
-    public static class Aggregator implements GeoPointFieldData.ValueInDocProc {
+    public static class Aggregator implements GeoPointValues.LatLonValueInDocProc {
 
         private final GeoDistance.FixedSourceDistance fixedSourceDistance;
 
@@ -115,6 +92,10 @@ public class GeoDistanceFacetCollector extends AbstractFacetCollector {
             this.entries = entries;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double lat, double lon) {
             double distance = fixedSourceDistance.calculate(lat, lon);
diff --git a/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetProcessor.java
index 9df9a8b2741..57404020491 100644
--- a/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacetProcessor.java
@@ -25,6 +25,9 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.search.geo.GeoDistance;
 import org.elasticsearch.index.search.geo.GeoHashUtils;
@@ -179,17 +182,28 @@ public class GeoDistanceFacetProcessor extends AbstractComponent implements Face
             lon = point.lon;
         }
 
+        FieldMapper keyFieldMapper = context.smartNameFieldMapper(fieldName);
+        if (keyFieldMapper == null) {
+            throw new FacetPhaseExecutionException(facetName, "failed to find mapping for [" + fieldName + "]");
+        }
+        IndexGeoPointFieldData keyIndexFieldData = context.fieldData().getForField(keyFieldMapper);
+
         if (valueFieldName != null) {
-            return new ValueGeoDistanceFacetCollector(facetName, fieldName, lat, lon, unit, geoDistance, entries.toArray(new GeoDistanceFacet.Entry[entries.size()]),
-                    context, valueFieldName);
+            FieldMapper valueFieldMapper = context.smartNameFieldMapper(valueFieldName);
+            if (valueFieldMapper == null) {
+                throw new FacetPhaseExecutionException(facetName, "failed to find mapping for [" + valueFieldName + "]");
+            }
+            IndexNumericFieldData valueIndexFieldData = context.fieldData().getForField(valueFieldMapper);
+            return new ValueGeoDistanceFacetCollector(facetName, keyIndexFieldData, lat, lon, unit, geoDistance, entries.toArray(new GeoDistanceFacet.Entry[entries.size()]),
+                    context, valueIndexFieldData);
         }
 
         if (valueScript != null) {
-            return new ScriptGeoDistanceFacetCollector(facetName, fieldName, lat, lon, unit, geoDistance, entries.toArray(new GeoDistanceFacet.Entry[entries.size()]),
+            return new ScriptGeoDistanceFacetCollector(facetName, keyIndexFieldData, lat, lon, unit, geoDistance, entries.toArray(new GeoDistanceFacet.Entry[entries.size()]),
                     context, scriptLang, valueScript, params);
         }
 
-        return new GeoDistanceFacetCollector(facetName, fieldName, lat, lon, unit, geoDistance, entries.toArray(new GeoDistanceFacet.Entry[entries.size()]),
+        return new GeoDistanceFacetCollector(facetName, keyIndexFieldData, lat, lon, unit, geoDistance, entries.toArray(new GeoDistanceFacet.Entry[entries.size()]),
                 context);
     }
 
diff --git a/src/main/java/org/elasticsearch/search/facet/geodistance/ScriptGeoDistanceFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/geodistance/ScriptGeoDistanceFacetCollector.java
index d3251a84509..d9d200b2016 100644
--- a/src/main/java/org/elasticsearch/search/facet/geodistance/ScriptGeoDistanceFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/geodistance/ScriptGeoDistanceFacetCollector.java
@@ -22,7 +22,8 @@ package org.elasticsearch.search.facet.geodistance;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.search.geo.GeoDistance;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.internal.SearchContext;
@@ -39,10 +40,10 @@ public class ScriptGeoDistanceFacetCollector extends GeoDistanceFacetCollector {
 
     private Aggregator scriptAggregator;
 
-    public ScriptGeoDistanceFacetCollector(String facetName, String fieldName, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
+    public ScriptGeoDistanceFacetCollector(String facetName, IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
                                            GeoDistanceFacet.Entry[] entries, SearchContext context,
                                            String scriptLang, String script, Map<String, Object> params) {
-        super(facetName, fieldName, lat, lon, unit, geoDistance, entries, context);
+        super(facetName, indexFieldData, lat, lon, unit, geoDistance, entries, context);
 
         this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
         this.aggregator = new Aggregator(fixedSourceDistance, entries);
@@ -67,7 +68,7 @@ public class ScriptGeoDistanceFacetCollector extends GeoDistanceFacetCollector {
         super.doCollect(doc);
     }
 
-    public static class Aggregator implements GeoPointFieldData.ValueInDocProc {
+    public static class Aggregator implements GeoPointValues.LatLonValueInDocProc {
 
         private final GeoDistance.FixedSourceDistance fixedSourceDistance;
 
@@ -80,6 +81,10 @@ public class ScriptGeoDistanceFacetCollector extends GeoDistanceFacetCollector {
             this.entries = entries;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double lat, double lon) {
             double distance = fixedSourceDistance.calculate(lat, lon);
diff --git a/src/main/java/org/elasticsearch/search/facet/geodistance/ValueGeoDistanceFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/geodistance/ValueGeoDistanceFacetCollector.java
index 45c94ef59a3..9784abf4baa 100644
--- a/src/main/java/org/elasticsearch/search/facet/geodistance/ValueGeoDistanceFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/geodistance/ValueGeoDistanceFacetCollector.java
@@ -21,12 +21,11 @@ package org.elasticsearch.search.facet.geodistance;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.search.geo.GeoDistance;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -36,35 +35,27 @@ import java.io.IOException;
  */
 public class ValueGeoDistanceFacetCollector extends GeoDistanceFacetCollector {
 
-    private final String indexValueFieldName;
+    private final IndexNumericFieldData valueIndexFieldData;
 
-    private final FieldDataType valueFieldDataType;
-
-    public ValueGeoDistanceFacetCollector(String facetName, String fieldName, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
-                                          GeoDistanceFacet.Entry[] entries, SearchContext context, String valueFieldName) {
-        super(facetName, fieldName, lat, lon, unit, geoDistance, entries, context);
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(valueFieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new FacetPhaseExecutionException(facetName, "No mapping found for field [" + valueFieldName + "]");
-        }
-        this.indexValueFieldName = smartMappers.mapper().names().indexName();
-        this.valueFieldDataType = smartMappers.mapper().fieldDataType();
+    public ValueGeoDistanceFacetCollector(String facetName, IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
+                                          GeoDistanceFacet.Entry[] entries, SearchContext context, IndexNumericFieldData valueIndexFieldData) {
+        super(facetName, indexFieldData, lat, lon, unit, geoDistance, entries, context);
+        this.valueIndexFieldData = valueIndexFieldData;
         this.aggregator = new Aggregator(fixedSourceDistance, entries);
     }
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
         super.doSetNextReader(context);
-        ((Aggregator) this.aggregator).valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, context.reader(), indexValueFieldName);
+        ((Aggregator) this.aggregator).valueValues = valueIndexFieldData.load(context).getDoubleValues();
     }
 
-    public static class Aggregator implements GeoPointFieldData.ValueInDocProc {
+    public static class Aggregator implements GeoPointValues.LatLonValueInDocProc {
 
         private final GeoDistance.FixedSourceDistance fixedSourceDistance;
         private final GeoDistanceFacet.Entry[] entries;
 
-        NumericFieldData valueFieldData;
+        DoubleValues valueValues;
 
         final ValueAggregator valueAggregator = new ValueAggregator();
 
@@ -73,6 +64,10 @@ public class ValueGeoDistanceFacetCollector extends GeoDistanceFacetCollector {
             this.entries = entries;
         }
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double lat, double lon) {
             double distance = fixedSourceDistance.calculate(lat, lon);
@@ -84,16 +79,20 @@ public class ValueGeoDistanceFacetCollector extends GeoDistanceFacetCollector {
                     entry.foundInDoc = true;
                     entry.count++;
                     valueAggregator.entry = entry;
-                    valueFieldData.forEachValueInDoc(docId, valueAggregator);
+                    valueValues.forEachValueInDoc(docId, valueAggregator);
                 }
             }
         }
     }
 
-    public static class ValueAggregator implements NumericFieldData.DoubleValueInDocProc {
+    public static class ValueAggregator implements DoubleValues.ValueInDocProc {
 
         GeoDistanceFacet.Entry entry;
 
+        @Override
+        public void onMissing(int docId) {
+        }
+
         @Override
         public void onValue(int docId, double value) {
             entry.totalCount++;

From fc09ce7ac90066e6034496541ee32f0d55be0b9e Mon Sep 17 00:00:00 2001
From: uboness <uboness@gmail.com>
Date: Sun, 20 Jan 2013 16:06:07 +0100
Subject: [PATCH 16/35] Implemented int field data

---
 .../index/fielddata/ByteValues.java           |  98 ++
 .../index/fielddata/DoubleValues.java         | 102 ++
 .../fielddata/IndexFieldDataService.java      |   7 +-
 .../index/fielddata/LongValues.java           |   3 +
 .../index/fielddata/ShortValues.java          |  98 ++
 .../index/fielddata/StringValues.java         |  95 ++
 .../plain/IntArrayAtomicFieldData.java        | 927 ++++++++++++++++++
 .../plain/IntArrayIndexFieldData.java         | 157 +++
 .../index/fielddata/IntFieldDataTests.java    | 107 ++
 9 files changed, 1590 insertions(+), 4 deletions(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/IntFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java b/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
index 86d4e885936..cebabaeb3bc 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata;
 
 import org.elasticsearch.ElasticSearchIllegalStateException;
 import org.elasticsearch.index.fielddata.util.ByteArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
 import org.elasticsearch.index.fielddata.util.LongArrayRef;
 
 /**
@@ -99,6 +100,103 @@ public interface ByteValues {
         }
     }
 
+    public static class IntBased implements ByteValues {
+
+        private final IntValues values;
+
+        private final ByteArrayRef arrayScratch = new ByteArrayRef(new byte[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public IntBased(IntValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public byte getValue(int docId) {
+            return (byte) values.getValue(docId);
+        }
+
+        @Override
+        public byte getValueMissing(int docId, byte missingValue) {
+            return (byte) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public ByteArrayRef getValues(int docId) {
+            IntArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return ByteArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (byte) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private IntValues.Iter iter;
+
+            public ValueIter reset(IntValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public byte next() {
+                return (byte) iter.next();
+            }
+        }
+
+        static class Proc implements IntValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, int value) {
+                proc.onValue(docId, (byte) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+
     public static class LongBased implements ByteValues {
 
         private final LongValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java b/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
index 3e3d595cb61..c7374d4f0a3 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
@@ -21,6 +21,8 @@ package org.elasticsearch.index.fielddata;
 
 import org.elasticsearch.ElasticSearchIllegalStateException;
 import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
 
 /**
  */
@@ -97,4 +99,104 @@ public interface DoubleValues {
             }
         }
     }
+
+    public static class LongBased implements DoubleValues {
+
+        private final LongValues values;
+        private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public double getValue(int docId) {
+            return (double) values.getValue(docId);
+        }
+
+        @Override
+        public double getValueMissing(int docId, double missingValue) {
+            if (!values.hasValue(docId)) {
+                return missingValue;
+            }
+            return getValue(docId);
+        }
+
+        @Override
+        public DoubleArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return DoubleArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (double) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return this.iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            private ValueIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public double next() {
+                return (double) iter.next();
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            private Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                this.proc.onValue(docId, (double) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                this.proc.onMissing(docId);
+            }
+        }
+
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index ed6b277b588..bf1775735f6 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -29,10 +29,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.plain.ConcreteBytesRefIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.GeoPointDoubleArrayIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.LongArrayIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.*;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
@@ -51,6 +48,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put("double", new DoubleArrayIndexFieldData.Builder())
                 .put("long", new LongArrayIndexFieldData.Builder())
                 .put("geo_point", new GeoPointDoubleArrayIndexFieldData.Builder())
+                .put("int", new IntArrayIndexFieldData.Builder())
                 .immutableMap();
 
         buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
@@ -58,6 +56,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("long", "array"), new LongArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder())
+                .put(Tuple.tuple("int", "array"), new IntArrayIndexFieldData.Builder())
                 .immutableMap();
     }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/LongValues.java b/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
index f6de0d87233..d275a7329f5 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.fielddata;
 
 import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
 import org.elasticsearch.index.fielddata.util.LongArrayRef;
 
 /**
@@ -47,6 +48,7 @@ public interface LongValues {
     void forEachValueInDoc(int docId, ValueInDocProc proc);
 
     static interface ValueInDocProc {
+
         void onValue(int docId, long value);
 
         void onMissing(int docId);
@@ -97,4 +99,5 @@ public interface LongValues {
             }
         }
     }
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java b/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
index 43de95e1369..0346ff78377 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.fielddata;
 
 import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
 import org.elasticsearch.index.fielddata.util.LongArrayRef;
 import org.elasticsearch.index.fielddata.util.ShortArrayRef;
 
@@ -99,6 +100,103 @@ public interface ShortValues {
         }
     }
 
+    public static class IntBased implements ShortValues {
+
+        private final IntValues values;
+
+        private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1);
+        private final ValueIter iter = new ValueIter();
+        private final Proc proc = new Proc();
+
+        public IntBased(IntValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public short getValue(int docId) {
+            return (short) values.getValue(docId);
+        }
+
+        @Override
+        public short getValueMissing(int docId, short missingValue) {
+            return (short) values.getValueMissing(docId, missingValue);
+        }
+
+        @Override
+        public ShortArrayRef getValues(int docId) {
+            IntArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) {
+                return ShortArrayRef.EMPTY;
+            }
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = (short) arrayRef.values[i];
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return iter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValueIter implements Iter {
+
+            private IntValues.Iter iter;
+
+            public ValueIter reset(IntValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public short next() {
+                return (short) iter.next();
+            }
+        }
+
+        static class Proc implements IntValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            public Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, int value) {
+                proc.onValue(docId, (short) value);
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+
     public static class LongBased implements ShortValues {
 
         private final LongValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
index d017568f328..56f1a5e08a4 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.fielddata;
 
 import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
 import org.elasticsearch.index.fielddata.util.StringArrayRef;
 
 /**
@@ -99,4 +100,98 @@ public interface StringValues {
             }
         }
     }
+
+    public static class LongBased implements StringValues {
+
+        private final LongValues values;
+
+        private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+        private final ValuesIter valuesIter = new ValuesIter();
+        private final Proc proc = new Proc();
+
+        public LongBased(LongValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public String getValue(int docId) {
+            if (!values.hasValue(docId)) {
+                return null;
+            }
+            return Long.toString(values.getValue(docId));
+        }
+
+        @Override
+        public StringArrayRef getValues(int docId) {
+            LongArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) return StringArrayRef.EMPTY;
+
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = Long.toString(arrayRef.values[i]);
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valuesIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValuesIter implements Iter {
+
+            private LongValues.Iter iter;
+
+            private ValuesIter reset(LongValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public String next() {
+                return Long.toString(iter.next());
+            }
+        }
+
+        static class Proc implements LongValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            private Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, long value) {
+                proc.onValue(docId, Long.toString(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
new file mode 100644
index 00000000000..774faefb941
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
@@ -0,0 +1,927 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData {
+
+    protected final int[] values;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public IntArrayAtomicFieldData(int[] values, int numDocs) {
+        this.values = values;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    public static class WithOrdinals extends IntArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(int[] values, int numDocs, Ordinals ordinals) {
+            super(values, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + +RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_INT) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericInteger(getIntValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, ordinals.ordinals());
+        }
+
+        static class IntValues implements org.elasticsearch.index.fielddata.IntValues {
+
+            private final int[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final IntArrayRef arrayScratch = new IntArrayRef(new int[1], 1);
+            private final ValuesIter iter;
+
+            IntValues(int[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public int getValue(int docId) {
+                return values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public int getValueMissing(int docId, int missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return values[ord];
+                }
+            }
+
+            @Override
+            public IntArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return IntArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final int[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(int[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public int next() {
+                    int value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final int[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final ValuesIter iter;
+
+            LongValues(int[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (long) values[ord];
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return LongArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (long) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final int[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(int[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public long next() {
+                    int value = values[ord];
+                    ord = ordsIter.next();
+                    return (long) value;
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final int[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final ValuesIter iter;
+
+            DoubleValues(int[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (double) values[ord];
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return DoubleArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (double) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (double) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final int[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(int[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public double next() {
+                    int value = values[ord];
+                    ord = ordsIter.next();
+                    return (double) value;
+                }
+            }
+        }
+    }
+
+    /**
+     * A single valued case, where not all values are "set", so we have a FixedBitSet that
+     * indicates which values have an actual value.
+     */
+    public static class SingleFixedSet extends IntArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(int[] values, int numDocs, FixedBitSet set) {
+            super(values, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues(values, set);
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, set);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, set);
+        }
+
+        static class IntValues implements org.elasticsearch.index.fielddata.IntValues {
+
+            private final int[] values;
+            private final FixedBitSet set;
+
+            private final IntArrayRef arrayScratch = new IntArrayRef(new int[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            IntValues(int[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public int getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public int getValueMissing(int docId, int missingValue) {
+                if (set.get(docId)) {
+                    return values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public IntArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = values[docId];
+                    return arrayScratch;
+                } else {
+                    return IntArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, values[docId]);
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final int[] values;
+            private final FixedBitSet set;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(int[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                if (set.get(docId)) {
+                    return (long) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (long) values[docId];
+                    return arrayScratch;
+                } else {
+                    return LongArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((long) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (long) values[docId]);
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final int[] values;
+            private final FixedBitSet set;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(int[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                if (set.get(docId)) {
+                    return (double) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (double) values[docId];
+                    return arrayScratch;
+                } else {
+                    return DoubleArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((double) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (double) values[docId]);
+                }
+            }
+        }
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends IntArrayAtomicFieldData {
+
+        /**
+         * Note, here, we assume that there is no offset by 1 from docId, so position 0
+         * is the value for docId 0.
+         */
+        public Single(int[] values, int numDocs) {
+            super(values, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues(values);
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values);
+        }
+
+        static class IntValues implements org.elasticsearch.index.fielddata.IntValues {
+
+            private final int[] values;
+
+            private final IntArrayRef arrayScratch = new IntArrayRef(new int[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            IntValues(int[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public int getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public int getValueMissing(int docId, int missingValue) {
+                return values[docId];
+            }
+
+            @Override
+            public IntArrayRef getValues(int docId) {
+                arrayScratch.values[0] = values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, values[docId]);
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final int[] values;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(int[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (long) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((long) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (long) values[docId]);
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final int[] values;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(int[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (double) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((double) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (double) values[docId]);
+            }
+        }
+
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
new file mode 100644
index 00000000000..f42baad69a0
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TIntArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class IntArrayIndexFieldData extends AbstractIndexFieldData<IntArrayAtomicFieldData> implements IndexNumericFieldData<IntArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new IntArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        }
+    }
+
+    public IntArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
+    }
+
+    @Override
+    public NumericType getNumericType() {
+        return NumericType.INT;
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public IntArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public IntArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldNames().indexName());
+        if (terms == null) {
+            return new IntArrayAtomicFieldData.Single(new int[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TIntArrayList values = new TIntArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add(0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add(FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            int[] sValues = new int[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sValues[i] = values.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new IntArrayAtomicFieldData.Single(sValues, reader.maxDoc());
+            } else {
+                return new IntArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new IntArrayAtomicFieldData.WithOrdinals(values.toArray(new int[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        return new DoubleValuesComparatorSource(this, missingValue);
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/IntFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/IntFieldDataTests.java
new file mode 100644
index 00000000000..a2b7955cd3e
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/IntFieldDataTests.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.document.*;
+import org.elasticsearch.index.fielddata.FieldDataType;
+
+/**
+ */
+public class IntFieldDataTests extends NumericFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("int");
+    }
+
+    @Override
+    protected void fillSingleValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new IntField("value", 2, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new IntField("value", 1, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new IntField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillSingleValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new IntField("value", 2, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new IntField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new IntField("value", 2, Field.Store.NO));
+        d.add(new IntField("value", 4, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new IntField("value", 1, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new IntField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new IntField("value", 2, Field.Store.NO));
+        d.add(new IntField("value", 4, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new IntField("value", 3, Field.Store.NO));
+        writer.addDocument(d);
+    }
+}

From d5e70a27df599c14b41a02226836a1348c4a3f9a Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 21:29:47 +0100
Subject: [PATCH 17/35] integer type to support int field data type

---
 .../org/elasticsearch/index/mapper/core/IntegerFieldMapper.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
index 39bf7acd986..9bbe52b1ee0 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
@@ -133,7 +133,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
 
     @Override
     public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        throw new ElasticSearchIllegalArgumentException("not implemented");
+        return new org.elasticsearch.index.fielddata.FieldDataType("int");
     }
 
     @Override

From 9137fcc6fc52e8f611699dd57f07557c27e9fd06 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 22:03:07 +0100
Subject: [PATCH 18/35] move geo distance sorting to use new field data

---
 .../GeoDistanceComparator.java                | 133 ++++++++++++++++++
 .../GeoDistanceComparatorSource.java          |  60 ++++++++
 .../search/sort/GeoDistanceSortParser.java    |  17 ++-
 3 files changed, 208 insertions(+), 2 deletions(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java
new file mode 100644
index 00000000000..b415a8cebf9
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
+import org.elasticsearch.index.search.geo.GeoDistance;
+
+import java.io.IOException;
+
+/**
+ */
+public class GeoDistanceComparator extends FieldComparator<Double> {
+
+    protected final IndexGeoPointFieldData indexFieldData;
+
+    protected final double lat;
+    protected final double lon;
+    protected final DistanceUnit unit;
+    protected final GeoDistance geoDistance;
+    protected final GeoDistance.FixedSourceDistance fixedSourceDistance;
+
+    private final double[] values;
+    private double bottom;
+
+    private GeoPointValues readerValues;
+
+    public GeoDistanceComparator(int numHits, IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance) {
+        this.values = new double[numHits];
+        this.indexFieldData = indexFieldData;
+        this.lat = lat;
+        this.lon = lon;
+        this.unit = unit;
+        this.geoDistance = geoDistance;
+        this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, unit);
+    }
+
+    @Override
+    public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getGeoPointValues();
+        return this;
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final double v1 = values[slot1];
+        final double v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public int compareBottom(int doc) {
+        double distance;
+        GeoPoint geoPoint = readerValues.getValue(doc);
+        if (geoPoint == null) {
+            // is this true? push this to the "end"
+            distance = Double.MAX_VALUE;
+        } else {
+            distance = fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon());
+        }
+        final double v2 = distance;
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Double distance2) throws IOException {
+        double distance1;
+        GeoPoint geoPoint = readerValues.getValue(doc);
+        if (geoPoint == null) {
+            // is this true? push this to the "end"
+            distance1 = Double.MAX_VALUE;
+        } else {
+            distance1 = fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon());
+        }
+        return (int) (distance1 - distance2);
+    }
+
+    @Override
+    public void copy(int slot, int doc) {
+        double distance;
+        GeoPoint geoPoint = readerValues.getValue(doc);
+        if (geoPoint == null) {
+            // is this true? push this to the "end"
+            distance = Double.MAX_VALUE;
+        } else {
+            distance = fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon());
+        }
+        values[slot] = distance;
+    }
+
+    @Override
+    public void setBottom(final int bottom) {
+        this.bottom = values[bottom];
+    }
+
+    @Override
+    public Double value(int slot) {
+        return values[slot];
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java
new file mode 100644
index 00000000000..c4266121fc9
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.search.geo.GeoDistance;
+
+import java.io.IOException;
+
+/**
+ */
+public class GeoDistanceComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexGeoPointFieldData indexFieldData;
+    private final double lat;
+    private final double lon;
+    private final DistanceUnit unit;
+    private final GeoDistance geoDistance;
+
+    public GeoDistanceComparatorSource(IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance) {
+        this.indexFieldData = indexFieldData;
+        this.lat = lat;
+        this.lon = lon;
+        this.unit = unit;
+        this.geoDistance = geoDistance;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.DOUBLE;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert indexFieldData.getFieldNames().indexName().equals(fieldname);
+        // TODO support multi value?
+        return new GeoDistanceComparator(numHits, indexFieldData, lat, lon, unit, geoDistance);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java
index be41bc03f55..9065467645a 100644
--- a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java
+++ b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java
@@ -20,10 +20,17 @@
 package org.elasticsearch.search.sort;
 
 import org.apache.lucene.search.SortField;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.fielddata.fieldcomparator.GeoDistanceComparatorSource;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
-import org.elasticsearch.index.search.geo.*;
+import org.elasticsearch.index.search.geo.GeoDistance;
+import org.elasticsearch.index.search.geo.GeoHashUtils;
+import org.elasticsearch.index.search.geo.GeoUtils;
+import org.elasticsearch.index.search.geo.Point;
 import org.elasticsearch.search.internal.SearchContext;
 
 /**
@@ -117,6 +124,12 @@ public class GeoDistanceSortParser implements SortParser {
             lon = point.lon;
         }
 
-        return new SortField(fieldName, GeoDistanceDataComparator.comparatorSource(fieldName, lat, lon, unit, geoDistance, context.fieldDataCache(), context.mapperService()), reverse);
+        FieldMapper mapper = context.smartNameFieldMapper(fieldName);
+        if (mapper == null) {
+            throw new ElasticSearchIllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
+        }
+        IndexGeoPointFieldData indexFieldData = context.fieldData().getForField(mapper);
+
+        return new SortField(fieldName, new GeoDistanceComparatorSource(indexFieldData, lat, lon, unit, geoDistance), reverse);
     }
 }

From 3840439365995595f262cd9505890289dcb95c81 Mon Sep 17 00:00:00 2001
From: uboness <uboness@gmail.com>
Date: Sun, 20 Jan 2013 22:08:46 +0100
Subject: [PATCH 19/35] float field data implementation

---
 .../fielddata/IndexFieldDataService.java      |   6 +-
 .../index/fielddata/StringValues.java         | 191 ++++
 .../plain/FloatArrayAtomicFieldData.java      | 925 ++++++++++++++++++
 .../plain/FloatArrayIndexFieldData.java       | 157 +++
 .../plain/IntArrayAtomicFieldData.java        |   2 +-
 .../index/mapper/core/FloatFieldMapper.java   |   2 +-
 .../index/fielddata/FloatFieldDataTests.java  | 127 +++
 7 files changed, 1406 insertions(+), 4 deletions(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index bf1775735f6..2032e59a725 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -45,18 +45,20 @@ public class IndexFieldDataService extends AbstractIndexComponent {
     static {
         buildersByType = MapBuilder.<String, IndexFieldData.Builder>newMapBuilder()
                 .put("string", new ConcreteBytesRefIndexFieldData.Builder())
+                .put("float", new FloatArrayIndexFieldData.Builder())
                 .put("double", new DoubleArrayIndexFieldData.Builder())
+                .put("int", new IntArrayIndexFieldData.Builder())
                 .put("long", new LongArrayIndexFieldData.Builder())
                 .put("geo_point", new GeoPointDoubleArrayIndexFieldData.Builder())
-                .put("int", new IntArrayIndexFieldData.Builder())
                 .immutableMap();
 
         buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
+                .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
+                .put(Tuple.tuple("int", "array"), new IntArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("long", "array"), new LongArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder())
-                .put(Tuple.tuple("int", "array"), new IntArrayIndexFieldData.Builder())
                 .immutableMap();
     }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
index 56f1a5e08a4..3178e25665a 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.index.fielddata;
 
 import org.elasticsearch.ElasticSearchIllegalStateException;
+import org.elasticsearch.index.fielddata.util.FloatArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
 import org.elasticsearch.index.fielddata.util.LongArrayRef;
 import org.elasticsearch.index.fielddata.util.StringArrayRef;
 
@@ -101,6 +103,100 @@ public interface StringValues {
         }
     }
 
+    public static class IntBased implements StringValues {
+
+        private final IntValues values;
+
+        private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+        private final ValuesIter valuesIter = new ValuesIter();
+        private final Proc proc = new Proc();
+
+        public IntBased(IntValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public String getValue(int docId) {
+            if (!values.hasValue(docId)) {
+                return null;
+            }
+            return Integer.toString(values.getValue(docId));
+        }
+
+        @Override
+        public StringArrayRef getValues(int docId) {
+            IntArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) return StringArrayRef.EMPTY;
+
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = Integer.toString(arrayRef.values[i]);
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valuesIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValuesIter implements Iter {
+
+            private IntValues.Iter iter;
+
+            private ValuesIter reset(IntValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public String next() {
+                return Integer.toString(iter.next());
+            }
+        }
+
+        static class Proc implements IntValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            private Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, int value) {
+                proc.onValue(docId, Integer.toString(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+
     public static class LongBased implements StringValues {
 
         private final LongValues values;
@@ -194,4 +290,99 @@ public interface StringValues {
             }
         }
     }
+
+    public static class FloatBased implements StringValues {
+
+        private final FloatValues values;
+
+        private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+        private final ValuesIter valuesIter = new ValuesIter();
+        private final Proc proc = new Proc();
+
+        public FloatBased(FloatValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public String getValue(int docId) {
+            if (!values.hasValue(docId)) {
+                return null;
+            }
+            return Float.toString(values.getValue(docId));
+        }
+
+        @Override
+        public StringArrayRef getValues(int docId) {
+            FloatArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) return StringArrayRef.EMPTY;
+
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = Float.toString(arrayRef.values[i]);
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valuesIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValuesIter implements Iter {
+
+            private FloatValues.Iter iter;
+
+            private ValuesIter reset(FloatValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public String next() {
+                return Float.toString(iter.next());
+            }
+        }
+
+        static class Proc implements FloatValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            private Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, float value) {
+                proc.onValue(docId, Float.toString(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
new file mode 100644
index 00000000000..765a967ecdc
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
@@ -0,0 +1,925 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.*;
+
+/**
+ */
+public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldData {
+
+    protected final float[] values;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public FloatArrayAtomicFieldData(float[] values, int numDocs) {
+        this.values = values;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    public static class WithOrdinals extends FloatArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(float[] values, int numDocs, Ordinals ordinals) {
+            super(values, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_FLOAT) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.FloatBased(getFloatValues());
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, ordinals.ordinals());
+        }
+
+        static class FloatValues implements org.elasticsearch.index.fielddata.FloatValues {
+
+            private final float[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final FloatArrayRef arrayScratch = new FloatArrayRef(new float[1], 1);
+            private final ValuesIter iter;
+
+            FloatValues(float[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public float getValue(int docId) {
+                return values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public float getValueMissing(int docId, float missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return values[ord];
+                }
+            }
+
+            @Override
+            public FloatArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return FloatArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final float[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(float[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public float next() {
+                    float value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final float[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final ValuesIter iter;
+
+            LongValues(float[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (long) values[ord];
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return LongArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (long) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final float[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(float[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public long next() {
+                    float value = values[ord];
+                    ord = ordsIter.next();
+                    return (long) value;
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final float[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final ValuesIter iter;
+
+            DoubleValues(float[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (double) values[ord];
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return DoubleArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (double) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (double) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final float[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(float[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public double next() {
+                    float value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    /**
+     * A single valued case, where not all values are "set", so we have a FixedBitSet that
+     * indicates which values have an actual value.
+     */
+    public static class SingleFixedSet extends FloatArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(float[] values, int numDocs, FixedBitSet set) {
+            super(values, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_FLOAT) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.FloatBased(getFloatValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, set);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues(values, set);
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, set);
+        }
+
+        static class FloatValues implements org.elasticsearch.index.fielddata.FloatValues {
+
+            private final float[] values;
+            private final FixedBitSet set;
+
+            private final FloatArrayRef arrayScratch = new FloatArrayRef(new float[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            FloatValues(float[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public float getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public float getValueMissing(int docId, float missingValue) {
+                if (set.get(docId)) {
+                    return values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public FloatArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = values[docId];
+                    return arrayScratch;
+                } else {
+                    return FloatArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, values[docId]);
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final float[] values;
+            private final FixedBitSet set;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(float[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                if (set.get(docId)) {
+                    return (long) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (long) values[docId];
+                    return arrayScratch;
+                } else {
+                    return LongArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((long) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (long) values[docId]);
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final float[] values;
+            private final FixedBitSet set;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(float[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                if (set.get(docId)) {
+                    return (double) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (double) values[docId];
+                    return arrayScratch;
+                } else {
+                    return DoubleArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((double) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (double) values[docId]);
+                }
+            }
+        }
+
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends FloatArrayAtomicFieldData {
+
+        /**
+         * Note, here, we assume that there is no offset by 1 from docId, so position 0
+         * is the value for docId 0.
+         */
+        public Single(float[] values, int numDocs) {
+            super(values, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_FLOAT);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.FloatBased(getFloatValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues(values);
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values);
+        }
+
+        static class FloatValues implements org.elasticsearch.index.fielddata.FloatValues {
+
+            private final float[] values;
+
+            private final FloatArrayRef arrayScratch = new FloatArrayRef(new float[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            FloatValues(float[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public float getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public float getValueMissing(int docId, float missingValue) {
+                return values[docId];
+            }
+
+            @Override
+            public FloatArrayRef getValues(int docId) {
+                arrayScratch.values[0] = values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, values[docId]);
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final float[] values;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(float[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (long) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((long) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (long) values[docId]);
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final float[] values;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(float[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (double) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((double) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (double) values[docId]);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
new file mode 100644
index 00000000000..6a921b8a7c1
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TFloatArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayAtomicFieldData> implements IndexNumericFieldData<FloatArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new FloatArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        }
+    }
+
+    public FloatArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
+    }
+
+    @Override
+    public NumericType getNumericType() {
+        return NumericType.FLOAT;
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public FloatArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public FloatArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldNames().indexName());
+        if (terms == null) {
+            return new FloatArrayAtomicFieldData.Single(new float[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TFloatArrayList values = new TFloatArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add(0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add(FieldCache.NUMERIC_UTILS_FLOAT_PARSER.parseFloat(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            float[] sValues = new float[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sValues[i] = values.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new FloatArrayAtomicFieldData.Single(sValues, reader.maxDoc());
+            } else {
+                return new FloatArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new FloatArrayAtomicFieldData.WithOrdinals(values.toArray(new float[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        return new DoubleValuesComparatorSource(this, missingValue);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
index 774faefb941..7c01d8a3df1 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
@@ -85,7 +85,7 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public StringValues getStringValues() {
-            return new StringValues.LongBased(getLongValues());
+            return new StringValues.IntBased(getIntValues());
         }
 
         @Override
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
index 0b7148a5db5..3f3253b9af0 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
@@ -131,7 +131,7 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
 
     @Override
     public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        throw new ElasticSearchIllegalArgumentException("not implemented");
+        return new org.elasticsearch.index.fielddata.FieldDataType("float");
     }
 
     @Override
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java
new file mode 100644
index 00000000000..70db22655ea
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.StringField;
+import org.elasticsearch.index.fielddata.FieldDataType;
+
+/**
+ */
+public class FloatFieldDataTests extends NumericFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("float");
+    }
+
+    protected String one() {
+        return "1.0";
+    }
+
+    protected String two() {
+        return "2.0";
+    }
+
+    protected String three() {
+        return "3.0";
+    }
+
+    protected String four() {
+        return "4.0";
+    }
+
+
+    @Override
+    protected void fillSingleValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new FloatField("value", 2.0f, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new FloatField("value", 1.0f, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new FloatField("value", 3.0f, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillSingleValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new FloatField("value", 2.0f, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new FloatField("value", 3.0f, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueAllSet() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new FloatField("value", 2.0f, Field.Store.NO));
+        d.add(new FloatField("value", 4.0f, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        d.add(new FloatField("value", 1.0f, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new FloatField("value", 3.0f, Field.Store.NO));
+        writer.addDocument(d);
+    }
+
+    @Override
+    protected void fillMultiValueWithMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        d.add(new FloatField("value", 2.0f, Field.Store.NO));
+        d.add(new FloatField("value", 4.0f, Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        d.add(new FloatField("value", 3.0f, Field.Store.NO));
+        writer.addDocument(d);
+    }
+}

From f1f3c241fd5dd03a326cfb595c7b04045cd9f181 Mon Sep 17 00:00:00 2001
From: uboness <uboness@gmail.com>
Date: Sun, 20 Jan 2013 22:40:31 +0100
Subject: [PATCH 20/35] short field data implementation

---
 .../fielddata/IndexFieldDataService.java      |   2 +
 .../index/fielddata/StringValues.java         |  99 +-
 .../plain/FloatArrayAtomicFieldData.java      |   2 +-
 .../plain/ShortArrayAtomicFieldData.java      | 928 ++++++++++++++++++
 .../plain/ShortArrayIndexFieldData.java       | 157 +++
 .../index/mapper/core/ShortFieldMapper.java   |   2 +-
 .../index/fielddata/ShortFieldDataTests.java  |  37 +
 7 files changed, 1221 insertions(+), 6 deletions(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index 2032e59a725..793f46da9ec 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -47,6 +47,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put("string", new ConcreteBytesRefIndexFieldData.Builder())
                 .put("float", new FloatArrayIndexFieldData.Builder())
                 .put("double", new DoubleArrayIndexFieldData.Builder())
+                .put("short", new ShortArrayIndexFieldData.Builder())
                 .put("int", new IntArrayIndexFieldData.Builder())
                 .put("long", new LongArrayIndexFieldData.Builder())
                 .put("geo_point", new GeoPointDoubleArrayIndexFieldData.Builder())
@@ -56,6 +57,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
                 .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
+                .put(Tuple.tuple("short", "array"), new ShortArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("int", "array"), new IntArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("long", "array"), new LongArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder())
diff --git a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
index 3178e25665a..577cd6f02b4 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
@@ -20,10 +20,7 @@
 package org.elasticsearch.index.fielddata;
 
 import org.elasticsearch.ElasticSearchIllegalStateException;
-import org.elasticsearch.index.fielddata.util.FloatArrayRef;
-import org.elasticsearch.index.fielddata.util.IntArrayRef;
-import org.elasticsearch.index.fielddata.util.LongArrayRef;
-import org.elasticsearch.index.fielddata.util.StringArrayRef;
+import org.elasticsearch.index.fielddata.util.*;
 
 /**
  */
@@ -103,6 +100,100 @@ public interface StringValues {
         }
     }
 
+    public static class ShortBased implements StringValues {
+
+        private final ShortValues values;
+
+        private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+        private final ValuesIter valuesIter = new ValuesIter();
+        private final Proc proc = new Proc();
+
+        public ShortBased(ShortValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public String getValue(int docId) {
+            if (!values.hasValue(docId)) {
+                return null;
+            }
+            return Short.toString(values.getValue(docId));
+        }
+
+        @Override
+        public StringArrayRef getValues(int docId) {
+            ShortArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) return StringArrayRef.EMPTY;
+
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = Short.toString(arrayRef.values[i]);
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valuesIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValuesIter implements Iter {
+
+            private ShortValues.Iter iter;
+
+            private ValuesIter reset(ShortValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public String next() {
+                return Short.toString(iter.next());
+            }
+        }
+
+        static class Proc implements ShortValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            private Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, short value) {
+                proc.onValue(docId, Short.toString(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+
     public static class IntBased implements StringValues {
 
         private final IntValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
index 765a967ecdc..afba43c9f3a 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
@@ -416,7 +416,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
                 public double next() {
                     float value = values[ord];
                     ord = ordsIter.next();
-                    return value;
+                    return (double) value;
                 }
             }
         }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
new file mode 100644
index 00000000000..236bb4d89b0
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
@@ -0,0 +1,928 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+import org.elasticsearch.index.fielddata.util.ShortArrayRef;
+
+/**
+ */
+public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldData {
+
+    protected final short[] values;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public ShortArrayAtomicFieldData(short[] values, int numDocs) {
+        this.values = values;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    public static class WithOrdinals extends ShortArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(short[] values, int numDocs, Ordinals ordinals) {
+            super(values, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_SHORT) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.ShortBased(getShortValues());
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, ordinals.ordinals());
+        }
+
+        static class ShortValues implements org.elasticsearch.index.fielddata.ShortValues {
+
+            private final short[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1);
+            private final ValuesIter iter;
+
+            ShortValues(short[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public short getValue(int docId) {
+                return values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public short getValueMissing(int docId, short missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return values[ord];
+                }
+            }
+
+            @Override
+            public ShortArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return ShortArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final short[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(short[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public short next() {
+                    short value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final short[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final ValuesIter iter;
+
+            LongValues(short[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (long) values[ord];
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return LongArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (long) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final short[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(short[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public long next() {
+                    short value = values[ord];
+                    ord = ordsIter.next();
+                    return (long) value;
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final short[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final ValuesIter iter;
+
+            DoubleValues(short[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (double) values[ord];
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return DoubleArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (double) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (double) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final short[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(short[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public double next() {
+                    short value = values[ord];
+                    ord = ordsIter.next();
+                    return (double) value;
+                }
+            }
+        }
+    }
+
+    /**
+     * A single valued case, where not all values are "set", so we have a FixedBitSet that
+     * indicates which values have an actual value.
+     */
+    public static class SingleFixedSet extends ShortArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(short[] values, int numDocs, FixedBitSet set) {
+            super(values, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_SHORT) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.ShortBased(getShortValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues(values, set);
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, set);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, set);
+        }
+
+        static class ShortValues implements org.elasticsearch.index.fielddata.ShortValues {
+
+            private final short[] values;
+            private final FixedBitSet set;
+
+            private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            ShortValues(short[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public short getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public short getValueMissing(int docId, short missingValue) {
+                if (set.get(docId)) {
+                    return values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public ShortArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = values[docId];
+                    return arrayScratch;
+                } else {
+                    return ShortArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, values[docId]);
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final short[] values;
+            private final FixedBitSet set;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(short[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                if (set.get(docId)) {
+                    return (long) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (long) values[docId];
+                    return arrayScratch;
+                } else {
+                    return LongArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((long) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (long) values[docId]);
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final short[] values;
+            private final FixedBitSet set;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(short[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                if (set.get(docId)) {
+                    return (double) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (double) values[docId];
+                    return arrayScratch;
+                } else {
+                    return DoubleArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((double) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (double) values[docId]);
+                }
+            }
+        }
+
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends ShortArrayAtomicFieldData {
+
+        /**
+         * Note, here, we assume that there is no offset by 1 from docId, so position 0
+         * is the value for docId 0.
+         */
+        public Single(short[] values, int numDocs) {
+            super(values, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_SHORT);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.ShortBased(getShortValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues(values);
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values);
+        }
+
+        static class ShortValues implements org.elasticsearch.index.fielddata.ShortValues {
+
+            private final short[] values;
+
+            private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            ShortValues(short[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public short getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public short getValueMissing(int docId, short missingValue) {
+                return values[docId];
+            }
+
+            @Override
+            public ShortArrayRef getValues(int docId) {
+                arrayScratch.values[0] = values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, values[docId]);
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final short[] values;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(short[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (long) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((long) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (long) values[docId]);
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final short[] values;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(short[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (double) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((double) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (double) values[docId]);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
new file mode 100644
index 00000000000..d1f7c107ac4
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TShortArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class ShortArrayIndexFieldData extends AbstractIndexFieldData<ShortArrayAtomicFieldData> implements IndexNumericFieldData<ShortArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new ShortArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        }
+    }
+
+    public ShortArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
+    }
+
+    @Override
+    public NumericType getNumericType() {
+        return NumericType.SHORT;
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public ShortArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public ShortArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldNames().indexName());
+        if (terms == null) {
+            return new ShortArrayAtomicFieldData.Single(new short[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TShortArrayList values = new TShortArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add((short) 0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add((short) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            short[] sValues = new short[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sValues[i] = values.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new ShortArrayAtomicFieldData.Single(sValues, reader.maxDoc());
+            } else {
+                return new ShortArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new ShortArrayAtomicFieldData.WithOrdinals(values.toArray(new short[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        return new DoubleValuesComparatorSource(this, missingValue);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
index 729aafc6894..7c410ad093c 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
@@ -133,7 +133,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
 
     @Override
     public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        throw new ElasticSearchIllegalArgumentException("not implemented");
+        return new org.elasticsearch.index.fielddata.FieldDataType("short");
     }
 
     @Override
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java
new file mode 100644
index 00000000000..6ca9a5a9eca
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ShortFieldDataTests.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.StringField;
+import org.elasticsearch.index.fielddata.FieldDataType;
+
+/**
+ */
+public class ShortFieldDataTests extends IntFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("short");
+    }
+
+}

From 855b64a8a738fe94bc792f73eade1bae26118b85 Mon Sep 17 00:00:00 2001
From: uboness <uboness@gmail.com>
Date: Sun, 20 Jan 2013 23:06:00 +0100
Subject: [PATCH 21/35] byte field data implementation

---
 .../fielddata/IndexFieldDataService.java      |   2 +
 .../index/fielddata/StringValues.java         |  94 ++
 .../plain/ByteArrayAtomicFieldData.java       | 928 ++++++++++++++++++
 .../plain/ByteArrayIndexFieldData.java        | 157 +++
 .../index/mapper/core/ByteFieldMapper.java    |   2 +-
 .../index/fielddata/ByteFieldDataTests.java   |  33 +
 6 files changed, 1215 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/ByteFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index 793f46da9ec..c5e5ef492b3 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -47,6 +47,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put("string", new ConcreteBytesRefIndexFieldData.Builder())
                 .put("float", new FloatArrayIndexFieldData.Builder())
                 .put("double", new DoubleArrayIndexFieldData.Builder())
+                .put("byte", new ByteArrayIndexFieldData.Builder())
                 .put("short", new ShortArrayIndexFieldData.Builder())
                 .put("int", new IntArrayIndexFieldData.Builder())
                 .put("long", new LongArrayIndexFieldData.Builder())
@@ -57,6 +58,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
                 .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
+                .put(Tuple.tuple("byte", "array"), new ByteArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("short", "array"), new ShortArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("int", "array"), new IntArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("long", "array"), new LongArrayIndexFieldData.Builder())
diff --git a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
index 577cd6f02b4..e0671bef496 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
@@ -100,6 +100,100 @@ public interface StringValues {
         }
     }
 
+    public static class ByteBased implements StringValues {
+
+        private final ByteValues values;
+
+        private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+        private final ValuesIter valuesIter = new ValuesIter();
+        private final Proc proc = new Proc();
+
+        public ByteBased(ByteValues values) {
+            this.values = values;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return values.isMultiValued();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return values.hasValue(docId);
+        }
+
+        @Override
+        public String getValue(int docId) {
+            if (!values.hasValue(docId)) {
+                return null;
+            }
+            return Byte.toString(values.getValue(docId));
+        }
+
+        @Override
+        public StringArrayRef getValues(int docId) {
+            ByteArrayRef arrayRef = values.getValues(docId);
+            int size = arrayRef.size();
+            if (size == 0) return StringArrayRef.EMPTY;
+
+            arrayScratch.reset(size);
+            for (int i = arrayRef.start; i < arrayRef.end; i++) {
+                arrayScratch.values[arrayScratch.end++] = Byte.toString(arrayRef.values[i]);
+            }
+            return arrayScratch;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return valuesIter.reset(values.getIter(docId));
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            values.forEachValueInDoc(docId, this.proc.reset(proc));
+        }
+
+        static class ValuesIter implements Iter {
+
+            private ByteValues.Iter iter;
+
+            private ValuesIter reset(ByteValues.Iter iter) {
+                this.iter = iter;
+                return this;
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iter.hasNext();
+            }
+
+            @Override
+            public String next() {
+                return Byte.toString(iter.next());
+            }
+        }
+
+        static class Proc implements ByteValues.ValueInDocProc {
+
+            private ValueInDocProc proc;
+
+            private Proc reset(ValueInDocProc proc) {
+                this.proc = proc;
+                return this;
+            }
+
+            @Override
+            public void onValue(int docId, byte value) {
+                proc.onValue(docId, Byte.toString(value));
+            }
+
+            @Override
+            public void onMissing(int docId) {
+                proc.onMissing(docId);
+            }
+        }
+    }
+
     public static class ShortBased implements StringValues {
 
         private final ShortValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
new file mode 100644
index 00000000000..3378424be14
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
@@ -0,0 +1,928 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.ByteArrayRef;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
+
+/**
+ */
+public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData {
+
+    protected final byte[] values;
+    private final int numDocs;
+
+    protected long size = -1;
+
+    public ByteArrayAtomicFieldData(byte[] values, int numDocs) {
+        this.values = values;
+        this.numDocs = numDocs;
+    }
+
+    @Override
+    public int getNumDocs() {
+        return numDocs;
+    }
+
+    public static class WithOrdinals extends ByteArrayAtomicFieldData {
+
+        private final Ordinals ordinals;
+
+        public WithOrdinals(byte[] values, int numDocs, Ordinals ordinals) {
+            super(values, numDocs);
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return ordinals.isMultiValued();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length) + ordinals.getMemorySizeInBytes();
+            }
+            return size;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.ByteBased(getByteValues());
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, ordinals.ordinals());
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, ordinals.ordinals());
+        }
+
+        static class ByteValues implements org.elasticsearch.index.fielddata.ByteValues {
+
+            private final byte[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final ByteArrayRef arrayScratch = new ByteArrayRef(new byte[1], 1);
+            private final ValuesIter iter;
+
+            ByteValues(byte[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public byte getValue(int docId) {
+                return values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public byte getValueMissing(int docId, byte missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return values[ord];
+                }
+            }
+
+            @Override
+            public ByteArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return ByteArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final byte[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(byte[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public byte next() {
+                    byte value = values[ord];
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final byte[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final ValuesIter iter;
+
+            LongValues(byte[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (long) values[ord];
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return LongArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (long) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final byte[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(byte[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public long next() {
+                    byte value = values[ord];
+                    ord = ordsIter.next();
+                    return (long) value;
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final byte[] values;
+            private final Ordinals.Docs ordinals;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final ValuesIter iter;
+
+            DoubleValues(byte[] values, Ordinals.Docs ordinals) {
+                this.values = values;
+                this.ordinals = ordinals;
+                this.iter = new ValuesIter(values);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return ordinals.isMultiValued();
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return ordinals.getOrd(docId) != 0;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[ordinals.getOrd(docId)];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    return missingValue;
+                } else {
+                    return (double) values[ord];
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return DoubleArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = (double) values[ords.values[i]];
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, (double) values[ord]);
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final byte[] values;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(byte[] values) {
+                    this.values = values;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public double next() {
+                    byte value = values[ord];
+                    ord = ordsIter.next();
+                    return (double) value;
+                }
+            }
+        }
+    }
+
+    /**
+     * A single valued case, where not all values are "set", so we have a FixedBitSet that
+     * indicates which values have an actual value.
+     */
+    public static class SingleFixedSet extends ByteArrayAtomicFieldData {
+
+        private final FixedBitSet set;
+
+        public SingleFixedSet(byte[] values, int numDocs, FixedBitSet set) {
+            super(values, numDocs);
+            this.set = set;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.ByteBased(getByteValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues(values, set);
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values, set);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values, set);
+        }
+
+        static class ByteValues implements org.elasticsearch.index.fielddata.ByteValues {
+
+            private final byte[] values;
+            private final FixedBitSet set;
+
+            private final ByteArrayRef arrayScratch = new ByteArrayRef(new byte[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            ByteValues(byte[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public byte getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public byte getValueMissing(int docId, byte missingValue) {
+                if (set.get(docId)) {
+                    return values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public ByteArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = values[docId];
+                    return arrayScratch;
+                } else {
+                    return ByteArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset(values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, values[docId]);
+                }
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final byte[] values;
+            private final FixedBitSet set;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(byte[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                if (set.get(docId)) {
+                    return (long) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (long) values[docId];
+                    return arrayScratch;
+                } else {
+                    return LongArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((long) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (long) values[docId]);
+                }
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final byte[] values;
+            private final FixedBitSet set;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(byte[] values, FixedBitSet set) {
+                this.values = values;
+                this.set = set;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return set.get(docId);
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                if (set.get(docId)) {
+                    return (double) values[docId];
+                } else {
+                    return missingValue;
+                }
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                if (set.get(docId)) {
+                    arrayScratch.values[0] = (double) values[docId];
+                    return arrayScratch;
+                } else {
+                    return DoubleArrayRef.EMPTY;
+                }
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                if (set.get(docId)) {
+                    return iter.reset((double) values[docId]);
+                } else {
+                    return Iter.Empty.INSTANCE;
+                }
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                if (set.get(docId)) {
+                    proc.onValue(docId, (double) values[docId]);
+                }
+            }
+        }
+
+    }
+
+    /**
+     * Assumes all the values are "set", and docId is used as the index to the value array.
+     */
+    public static class Single extends ByteArrayAtomicFieldData {
+
+        /**
+         * Note, here, we assume that there is no offset by 1 from docId, so position 0
+         * is the value for docId 0.
+         */
+        public Single(byte[] values, int numDocs) {
+            super(values, numDocs);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            if (size == -1) {
+                size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length);
+            }
+            return size;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return new ScriptDocValues.NumericDouble(getDoubleValues());
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return new BytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.StringBased(getStringValues());
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return new StringValues.ByteBased(getByteValues());
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return new ByteValues(values);
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return new ShortValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return new IntValues.LongBased(getLongValues());
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return new LongValues(values);
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return new FloatValues.DoubleBased(getDoubleValues());
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return new DoubleValues(values);
+        }
+
+        static class ByteValues implements org.elasticsearch.index.fielddata.ByteValues {
+
+            private final byte[] values;
+
+            private final ByteArrayRef arrayScratch = new ByteArrayRef(new byte[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            ByteValues(byte[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public byte getValue(int docId) {
+                return values[docId];
+            }
+
+            @Override
+            public byte getValueMissing(int docId, byte missingValue) {
+                return values[docId];
+            }
+
+            @Override
+            public ByteArrayRef getValues(int docId) {
+                arrayScratch.values[0] = values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, values[docId]);
+            }
+        }
+
+        static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
+
+            private final byte[] values;
+
+            private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            LongValues(byte[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public long getValue(int docId) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public long getValueMissing(int docId, long missingValue) {
+                return (long) values[docId];
+            }
+
+            @Override
+            public LongArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (long) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((long) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (long) values[docId]);
+            }
+        }
+
+        static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
+
+            private final byte[] values;
+
+            private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            DoubleValues(byte[] values) {
+                this.values = values;
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public boolean hasValue(int docId) {
+                return true;
+            }
+
+            @Override
+            public double getValue(int docId) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public double getValueMissing(int docId, double missingValue) {
+                return (double) values[docId];
+            }
+
+            @Override
+            public DoubleArrayRef getValues(int docId) {
+                arrayScratch.values[0] = (double) values[docId];
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset((double) values[docId]);
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                proc.onValue(docId, (double) values[docId]);
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
new file mode 100644
index 00000000000..c4694089a84
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import gnu.trove.list.array.TByteArrayList;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class ByteArrayIndexFieldData extends AbstractIndexFieldData<ByteArrayAtomicFieldData> implements IndexNumericFieldData<ByteArrayAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new ByteArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        }
+    }
+
+    public ByteArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
+    }
+
+    @Override
+    public NumericType getNumericType() {
+        return NumericType.BYTE;
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public ByteArrayAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public ByteArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldNames().indexName());
+        if (terms == null) {
+            return new ByteArrayAtomicFieldData.Single(new byte[0], 0);
+        }
+
+        // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
+        final TByteArrayList values = new TByteArrayList();
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        values.add((byte) 0); // first "t" indicates null value
+        int termOrd = 1;  // current term number
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                values.add((byte) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term));
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        if (ordinals.size() == 1) {
+            int[] nativeOrdinals = ordinals.get(0);
+            FixedBitSet set = new FixedBitSet(reader.maxDoc());
+            byte[] sValues = new byte[reader.maxDoc()];
+            boolean allHaveValue = true;
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                int nativeOrdinal = nativeOrdinals[i];
+                if (nativeOrdinal == 0) {
+                    allHaveValue = false;
+                } else {
+                    set.set(i);
+                    sValues[i] = values.get(nativeOrdinal);
+                }
+            }
+            if (allHaveValue) {
+                return new ByteArrayAtomicFieldData.Single(sValues, reader.maxDoc());
+            } else {
+                return new ByteArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
+            }
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new ByteArrayAtomicFieldData.WithOrdinals(values.toArray(new byte[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        return new DoubleValuesComparatorSource(this, missingValue);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
index e62fea95626..386907e6800 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
@@ -130,7 +130,7 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
 
     @Override
     public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        throw new ElasticSearchIllegalArgumentException("not implemented");
+        return new org.elasticsearch.index.fielddata.FieldDataType("byte");
     }
 
     @Override
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/ByteFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ByteFieldDataTests.java
new file mode 100644
index 00000000000..6349da909e8
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/ByteFieldDataTests.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import org.elasticsearch.index.fielddata.FieldDataType;
+
+/**
+ */
+public class ByteFieldDataTests extends IntFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("byte");
+    }
+
+}

From 45f27fe96a1a52e1fb86c9a130aa5565a45809d9 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Sun, 20 Jan 2013 23:34:02 +0100
Subject: [PATCH 22/35] add packed bytes variant for strings/bytes

---
 .../fielddata/IndexFieldDataService.java      |   1 +
 .../plain/PackedBytesAtomicFieldData.java     | 750 ++++++++++++++++++
 .../plain/PackedBytesIndexFieldData.java      | 193 +++++
 .../PackedBytesStringFieldDataTests.java      |  35 +
 4 files changed, 979 insertions(+)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesAtomicFieldData.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesIndexFieldData.java
 create mode 100644 src/test/java/org/elasticsearch/test/unit/index/fielddata/PackedBytesStringFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index c5e5ef492b3..2cb4d966f9b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -56,6 +56,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
 
         buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
+                .put(Tuple.tuple("string", "packed_bytes"), new PackedBytesIndexFieldData.Builder())
                 .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("byte", "array"), new ByteArrayIndexFieldData.Builder())
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesAtomicFieldData.java
new file mode 100644
index 00000000000..0e251617e25
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesAtomicFieldData.java
@@ -0,0 +1,750 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.PackedInts;
+import org.elasticsearch.common.RamUsage;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
+import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.StringArrayRef;
+
+/**
+ */
+public class PackedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
+
+    // 0 ordinal in values means no value (its null)
+    private final PagedBytes.Reader bytes;
+    private final PackedInts.Reader termOrdToBytesOffset;
+    private final Ordinals ordinals;
+
+    private int[] hashes;
+    private long size = -1;
+
+    public PackedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals ordinals) {
+        this.bytes = bytes;
+        this.termOrdToBytesOffset = termOrdToBytesOffset;
+        this.ordinals = ordinals;
+    }
+
+    @Override
+    public boolean isMultiValued() {
+        return ordinals.isMultiValued();
+    }
+
+    @Override
+    public int getNumDocs() {
+        return ordinals.getNumDocs();
+    }
+
+    @Override
+    public boolean isValuesOrdered() {
+        return true;
+    }
+
+    @Override
+    public long getMemorySizeInBytes() {
+        if (size == -1) {
+            long size = ordinals.getMemorySizeInBytes();
+            // PackedBytes
+            size += RamUsage.NUM_BYTES_ARRAY_HEADER + bytes.getBlocks().length;
+            for (byte[] b : bytes.getBlocks()) {
+                size += b.length;
+            }
+            // PackedInts
+            size += termOrdToBytesOffset.ramBytesUsed();
+            this.size = size;
+        }
+        return size;
+    }
+
+    @Override
+    public OrdinalsBytesValues getBytesValues() {
+        return ordinals.isMultiValued() ? new BytesValues.Multi(bytes, termOrdToBytesOffset, ordinals.ordinals()) : new BytesValues.Single(bytes, termOrdToBytesOffset, ordinals.ordinals());
+    }
+
+    @Override
+    public OrdinalsHashedBytesValues getHashedBytesValues() {
+        if (hashes == null) {
+            int numberOfValues = termOrdToBytesOffset.size();
+            int[] hashes = new int[numberOfValues];
+            BytesRef scratch = new BytesRef();
+            for (int i = 0; i < numberOfValues; i++) {
+                BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(i));
+                hashes[i] = value == null ? 0 : value.hashCode();
+            }
+            this.hashes = hashes;
+        }
+        return ordinals.isMultiValued() ? new HashedBytesValues.Multi(bytes, termOrdToBytesOffset, hashes, ordinals.ordinals()) : new HashedBytesValues.Single(bytes, termOrdToBytesOffset, hashes, ordinals.ordinals());
+    }
+
+    @Override
+    public OrdinalsStringValues getStringValues() {
+        return ordinals.isMultiValued() ? new StringValues.Multi(bytes, termOrdToBytesOffset, ordinals.ordinals()) : new StringValues.Single(bytes, termOrdToBytesOffset, ordinals.ordinals());
+    }
+
+    @Override
+    public ScriptDocValues.Strings getScriptValues() {
+        return new ScriptDocValues.Strings(getStringValues());
+    }
+
+    static abstract class BytesValues implements org.elasticsearch.index.fielddata.OrdinalsBytesValues {
+
+        protected final PagedBytes.Reader bytes;
+        protected final PackedInts.Reader termOrdToBytesOffset;
+        protected final Ordinals.Docs ordinals;
+
+        protected final BytesRef scratch = new BytesRef();
+
+        BytesValues(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
+            this.bytes = bytes;
+            this.termOrdToBytesOffset = termOrdToBytesOffset;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return this.ordinals;
+        }
+
+        @Override
+        public BytesRef getValueByOrd(int ord) {
+            return bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+        }
+
+        @Override
+        public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
+            return bytes.fill(ret, termOrdToBytesOffset.get(ord));
+        }
+
+        @Override
+        public BytesRef getSafeValueByOrd(int ord) {
+            return bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord));
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public BytesRef makeSafe(BytesRef bytes) {
+            return BytesRef.deepCopyOf(bytes);
+        }
+
+        @Override
+        public BytesRef getValue(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+        }
+
+        @Override
+        public BytesRef getValueScratch(int docId, BytesRef ret) {
+            return bytes.fill(ret, termOrdToBytesOffset.get(ordinals.getOrd(docId)));
+        }
+
+        @Override
+        public BytesRef getValueSafe(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord));
+        }
+
+        static class Single extends BytesValues {
+
+            private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
+                super(bytes, termOrdToBytesOffset, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public BytesRefArrayRef getValues(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return BytesRefArrayRef.EMPTY;
+                arrayScratch.values[0] = bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord));
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(bytes.fill(scratch, termOrdToBytesOffset.get(ord)));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, bytes.fill(scratch, termOrdToBytesOffset.get(ord)));
+                }
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)));
+                }
+            }
+        }
+
+        static class Multi extends BytesValues {
+
+            private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[10], 0);
+            private final ValuesIter iter;
+            private final SafeValuesIter safeIter;
+
+            Multi(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
+                super(bytes, termOrdToBytesOffset, ordinals);
+                this.iter = new ValuesIter(bytes, termOrdToBytesOffset);
+                this.safeIter = new SafeValuesIter(bytes, termOrdToBytesOffset);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public BytesRefArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return BytesRefArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    arrayScratch.values[arrayScratch.end++] = bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ords.values[i]));
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return safeIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, bytes.fill(scratch, termOrdToBytesOffset.get(ord)));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final PagedBytes.Reader bytes;
+                private final PackedInts.Reader termOrdToBytesOffset;
+                private final BytesRef scratch = new BytesRef();
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset) {
+                    this.bytes = bytes;
+                    this.termOrdToBytesOffset = termOrdToBytesOffset;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public BytesRef next() {
+                    BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+
+            static class SafeValuesIter implements Iter {
+
+                private final PagedBytes.Reader bytes;
+                private final PackedInts.Reader termOrdToBytesOffset;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                SafeValuesIter(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset) {
+                    this.bytes = bytes;
+                    this.termOrdToBytesOffset = termOrdToBytesOffset;
+                }
+
+                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public BytesRef next() {
+                    BytesRef value = bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord));
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    static abstract class HashedBytesValues implements org.elasticsearch.index.fielddata.OrdinalsHashedBytesValues {
+
+        protected final PagedBytes.Reader bytes;
+        protected final PackedInts.Reader termOrdToBytesOffset;
+        protected final int[] hashes;
+        protected final Ordinals.Docs ordinals;
+
+        protected final BytesRef scratch1 = new BytesRef();
+        protected final HashedBytesRef scratch = new HashedBytesRef();
+
+        HashedBytesValues(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, int[] hashes, Ordinals.Docs ordinals) {
+            this.bytes = bytes;
+            this.termOrdToBytesOffset = termOrdToBytesOffset;
+            this.hashes = hashes;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return this.ordinals;
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            return scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]);
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]);
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public HashedBytesRef makeSafe(HashedBytesRef bytes) {
+            return new HashedBytesRef(BytesRef.deepCopyOf(bytes.bytes), bytes.hash);
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]);
+        }
+
+        @Override
+        public HashedBytesRef getValueSafe(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            return new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]);
+        }
+
+        static class Single extends HashedBytesValues {
+
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, int[] hashes, Ordinals.Docs ordinals) {
+                super(bytes, termOrdToBytesOffset, hashes, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]));
+                }
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                } else {
+                    proc.onValue(docId, new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]));
+                }
+            }
+        }
+
+        static class Multi extends HashedBytesValues {
+
+            private final ValuesIter iter;
+            private final SafeValuesIter safeIter;
+
+            Multi(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, int[] hashes, Ordinals.Docs ordinals) {
+                super(bytes, termOrdToBytesOffset, hashes, ordinals);
+                this.iter = new ValuesIter(bytes, termOrdToBytesOffset, hashes);
+                this.safeIter = new SafeValuesIter(bytes, termOrdToBytesOffset, hashes);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public Iter getIterSafe(int docId) {
+                return safeIter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            @Override
+            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    proc.onValue(docId, new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]));
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements Iter {
+
+                private final PagedBytes.Reader bytes;
+                private final PackedInts.Reader termOrdToBytesOffset;
+                private final int[] hashes;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                private final BytesRef scratch1 = new BytesRef();
+                private final HashedBytesRef scratch = new HashedBytesRef();
+
+                ValuesIter(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, int[] hashes) {
+                    this.bytes = bytes;
+                    this.termOrdToBytesOffset = termOrdToBytesOffset;
+                    this.hashes = hashes;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public HashedBytesRef next() {
+                    HashedBytesRef value = scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+
+            static class SafeValuesIter implements Iter {
+
+                private final PagedBytes.Reader bytes;
+                private final PackedInts.Reader termOrdToBytesOffset;
+                private final int[] hashes;
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                SafeValuesIter(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, int[] hashes) {
+                    this.bytes = bytes;
+                    this.termOrdToBytesOffset = termOrdToBytesOffset;
+                    this.hashes = hashes;
+                }
+
+                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public HashedBytesRef next() {
+                    HashedBytesRef value = new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]);
+                    ord = ordsIter.next();
+                    return value;
+                }
+            }
+        }
+    }
+
+    static abstract class StringValues implements OrdinalsStringValues {
+
+        protected final PagedBytes.Reader bytes;
+        protected final PackedInts.Reader termOrdToBytesOffset;
+        protected final Ordinals.Docs ordinals;
+
+        protected final BytesRef scratch = new BytesRef();
+
+        protected StringValues(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
+            this.bytes = bytes;
+            this.termOrdToBytesOffset = termOrdToBytesOffset;
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return ordinals;
+        }
+
+        @Override
+        public String getValueByOrd(int ord) {
+            BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+            return value.utf8ToString();
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return ordinals.getOrd(docId) != 0;
+        }
+
+        @Override
+        public String getValue(int docId) {
+            int ord = ordinals.getOrd(docId);
+            if (ord == 0) return null;
+            BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+            return value.utf8ToString();
+        }
+
+        static class Single extends StringValues {
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
+            private final Iter.Single iter = new Iter.Single();
+
+            Single(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
+                super(bytes, termOrdToBytesOffset, ordinals);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return false;
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return StringArrayRef.EMPTY;
+                BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+                arrayScratch.values[0] = value == null ? null : value.utf8ToString();
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) return Iter.Empty.INSTANCE;
+                return iter.reset(bytes.fill(scratch, termOrdToBytesOffset.get(ord)).utf8ToString());
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                int ord = ordinals.getOrd(docId);
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                proc.onValue(docId, bytes.fill(scratch, termOrdToBytesOffset.get(ord)).utf8ToString());
+            }
+        }
+
+        static class Multi extends StringValues {
+
+            private final StringArrayRef arrayScratch = new StringArrayRef(new String[10], 0);
+            private final ValuesIter iter;
+
+            Multi(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
+                super(bytes, termOrdToBytesOffset, ordinals);
+                iter = new ValuesIter(bytes, termOrdToBytesOffset);
+            }
+
+            @Override
+            public boolean isMultiValued() {
+                return true;
+            }
+
+            @Override
+            public StringArrayRef getValues(int docId) {
+                IntArrayRef ords = ordinals.getOrds(docId);
+                int size = ords.size();
+                if (size == 0) return StringArrayRef.EMPTY;
+
+                arrayScratch.reset(size);
+                for (int i = ords.start; i < ords.end; i++) {
+                    BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(ords.values[i]));
+                    arrayScratch.values[arrayScratch.end++] = value == null ? null : value.utf8ToString();
+                }
+                return arrayScratch;
+            }
+
+            @Override
+            public Iter getIter(int docId) {
+                return iter.reset(ordinals.getIter(docId));
+            }
+
+            @Override
+            public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
+                int ord = iter.next();
+                if (ord == 0) {
+                    proc.onMissing(docId);
+                    return;
+                }
+                do {
+                    BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+                    proc.onValue(docId, value == null ? null : value.utf8ToString());
+                } while ((ord = iter.next()) != 0);
+            }
+
+            static class ValuesIter implements StringValues.Iter {
+
+                private final PagedBytes.Reader bytes;
+                private final PackedInts.Reader termOrdToBytesOffset;
+                private final BytesRef scratch = new BytesRef();
+                private Ordinals.Docs.Iter ordsIter;
+                private int ord;
+
+                ValuesIter(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset) {
+                    this.bytes = bytes;
+                    this.termOrdToBytesOffset = termOrdToBytesOffset;
+                }
+
+                public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
+                    this.ordsIter = ordsIter;
+                    this.ord = ordsIter.next();
+                    return this;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return ord != 0;
+                }
+
+                @Override
+                public String next() {
+                    BytesRef value = bytes.fill(scratch, termOrdToBytesOffset.get(ord));
+                    ord = ordsIter.next();
+                    return value == null ? null : value.utf8ToString();
+                }
+            }
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesIndexFieldData.java
new file mode 100644
index 00000000000..fb70a1f000f
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesIndexFieldData.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.GrowableWriter;
+import org.apache.lucene.util.packed.PackedInts;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
+import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
+import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.util.ArrayList;
+
+/**
+ */
+public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedBytesAtomicFieldData> implements IndexOrdinalFieldData<PackedBytesAtomicFieldData> {
+
+    public static class Builder implements IndexFieldData.Builder {
+
+        @Override
+        public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
+            return new PackedBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        }
+    }
+
+    public PackedBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        return true;
+    }
+
+    @Override
+    public PackedBytesAtomicFieldData load(AtomicReaderContext context) {
+        try {
+            return cache.load(context, this);
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    @Override
+    public PackedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+        AtomicReader reader = context.reader();
+
+        Terms terms = reader.terms(getFieldNames().indexName());
+        if (terms == null) {
+            final PagedBytes bytes = new PagedBytes(1);
+            // 0 is reserved for "unset"
+            bytes.copyUsingLengthPrefix(new BytesRef());
+            GrowableWriter termOrdToBytesOffset = new GrowableWriter(1, 2, PackedInts.FASTEST);
+            return new PackedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
+        }
+
+        final PagedBytes bytes = new PagedBytes(15);
+        int startBytesBPV;
+        int startTermsBPV;
+        int startNumUniqueTerms;
+
+        int maxDoc = reader.maxDoc();
+        final int termCountHardLimit;
+        if (maxDoc == Integer.MAX_VALUE) {
+            termCountHardLimit = Integer.MAX_VALUE;
+        } else {
+            termCountHardLimit = maxDoc + 1;
+        }
+
+        // Try for coarse estimate for number of bits; this
+        // should be an underestimate most of the time, which
+        // is fine -- GrowableWriter will reallocate as needed
+        long numUniqueTerms = terms.size();
+        if (numUniqueTerms != -1L) {
+            if (numUniqueTerms > termCountHardLimit) {
+                // app is misusing the API (there is more than
+                // one term per doc); in this case we make best
+                // effort to load what we can (see LUCENE-2142)
+                numUniqueTerms = termCountHardLimit;
+            }
+
+            startBytesBPV = PackedInts.bitsRequired(numUniqueTerms * 4);
+            startTermsBPV = PackedInts.bitsRequired(numUniqueTerms);
+
+            startNumUniqueTerms = (int) numUniqueTerms;
+        } else {
+            startBytesBPV = 1;
+            startTermsBPV = 1;
+            startNumUniqueTerms = 1;
+        }
+
+        // TODO: expose this as an option..., have a nice parser for it...
+        float acceptableOverheadRatio = PackedInts.FAST;
+
+        GrowableWriter termOrdToBytesOffset = new GrowableWriter(startBytesBPV, 1 + startNumUniqueTerms, acceptableOverheadRatio);
+
+        ArrayList<int[]> ordinals = new ArrayList<int[]>();
+        int[] idx = new int[reader.maxDoc()];
+        ordinals.add(new int[reader.maxDoc()]);
+
+        // 0 is reserved for "unset"
+        bytes.copyUsingLengthPrefix(new BytesRef());
+        int termOrd = 1;
+
+        TermsEnum termsEnum = terms.iterator(null);
+        try
+
+        {
+            DocsEnum docsEnum = null;
+            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
+                if (termOrd == termOrdToBytesOffset.size()) {
+                    // NOTE: this code only runs if the incoming
+                    // reader impl doesn't implement
+                    // size (which should be uncommon)
+                    termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1 + termOrd, 1));
+                }
+                termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
+
+                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
+                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
+                    int[] ordinal;
+                    if (idx[docId] >= ordinals.size()) {
+                        ordinal = new int[reader.maxDoc()];
+                        ordinals.add(ordinal);
+                    } else {
+                        ordinal = ordinals.get(idx[docId]);
+                    }
+                    ordinal[docId] = termOrd;
+                    idx[docId]++;
+                }
+                termOrd++;
+            }
+        } catch (RuntimeException e) {
+            if (e.getClass().getName().endsWith("StopFillCacheException")) {
+                // all is well, in case numeric parsers are used.
+            } else {
+                throw e;
+            }
+        }
+
+        PagedBytes.Reader bytesReader = bytes.freeze(true);
+        PackedInts.Reader termOrdToBytesOffsetReader = termOrdToBytesOffset.getMutable();
+
+        if (ordinals.size() == 1) {
+            return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new SingleArrayOrdinals(ordinals.get(0), termOrd));
+        } else {
+            int[][] nativeOrdinals = new int[ordinals.size()][];
+            for (int i = 0; i < nativeOrdinals.length; i++) {
+                nativeOrdinals[i] = ordinals.get(i);
+            }
+            return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+        }
+
+    }
+
+    @Override
+    public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
+        // TODO support "missingValue" for sortMissingValue options here...
+        return new BytesRefFieldComparatorSource(this);
+    }
+}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/PackedBytesStringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/PackedBytesStringFieldDataTests.java
new file mode 100644
index 00000000000..8dfdd1030f1
--- /dev/null
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/PackedBytesStringFieldDataTests.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.fielddata;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.index.fielddata.FieldDataType;
+import org.testng.annotations.Test;
+
+/**
+ */
+@Test
+public class PackedBytesStringFieldDataTests extends StringFieldDataTests {
+
+    @Override
+    protected FieldDataType getFieldDataType() {
+        return new FieldDataType("string", "packed_bytes", ImmutableMap.<String, String>of());
+    }
+}

From b739bf97d4b4d957e1ea7c1b7f76145f66571095 Mon Sep 17 00:00:00 2001
From: uboness <uboness@gmail.com>
Date: Mon, 21 Jan 2013 00:39:25 +0100
Subject: [PATCH 23/35] added missing dedicated value comparators for the
 different indices field data

---
 .../fieldcomparator/ByteValuesComparator.java | 105 ++++++++++++++++++
 .../ByteValuesComparatorSource.java           |  62 +++++++++++
 .../FloatValuesComparator.java                | 105 ++++++++++++++++++
 .../FloatValuesComparatorSource.java          |  62 +++++++++++
 .../fieldcomparator/IntValuesComparator.java  | 105 ++++++++++++++++++
 .../IntValuesComparatorSource.java            |  62 +++++++++++
 .../fieldcomparator/LongValuesComparator.java | 105 ++++++++++++++++++
 .../LongValuesComparatorSource.java           |  62 +++++++++++
 .../ShortValuesComparator.java                | 105 ++++++++++++++++++
 .../ShortValuesComparatorSource.java          |  62 +++++++++++
 .../plain/ByteArrayIndexFieldData.java        |   4 +-
 .../plain/FloatArrayIndexFieldData.java       |   4 +-
 .../plain/IntArrayIndexFieldData.java         |   4 +-
 .../plain/LongArrayIndexFieldData.java        |   4 +-
 .../plain/ShortArrayIndexFieldData.java       |   4 +-
 15 files changed, 845 insertions(+), 10 deletions(-)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparatorSource.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparator.java
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparatorSource.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparator.java
new file mode 100644
index 00000000000..a96d678ac89
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.index.fielddata.ByteValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class ByteValuesComparator extends FieldComparator<Byte> {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final byte missingValue;
+
+    protected final byte[] values;
+    private byte bottom;
+    private ByteValues readerValues;
+
+    public ByteValuesComparator(IndexNumericFieldData indexFieldData, byte missingValue, int numHits) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+        this.values = new byte[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final byte v1 = values[slot1];
+        final byte v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void setBottom(int slot) {
+        this.bottom = values[slot];
+    }
+
+    @Override
+    public int compareBottom(int doc) throws IOException {
+        byte v2 = readerValues.getValueMissing(doc, missingValue);
+
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void copy(int slot, int doc) throws IOException {
+        values[slot] = readerValues.getValueMissing(doc, missingValue);
+    }
+
+    @Override
+    public FieldComparator<Byte> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getByteValues();
+        return this;
+    }
+
+    @Override
+    public Byte value(int slot) {
+        return Byte.valueOf(values[slot]);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Byte valueObj) throws IOException {
+        final byte value = valueObj.byteValue();
+        byte docValue = readerValues.getValueMissing(doc, missingValue);
+        if (docValue < value) {
+            return -1;
+        } else if (docValue > value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparatorSource.java
new file mode 100644
index 00000000000..c4a81be3c8d
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ByteValuesComparatorSource.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class ByteValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final Object missingValue;
+
+    public ByteValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.INT;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+
+        byte dMissingValue;
+        if (missingValue == null || "_last".equals(missingValue)) {
+            dMissingValue = reversed ? Byte.MIN_VALUE : Byte.MAX_VALUE;
+        } else if ("_first".equals(missingValue)) {
+            dMissingValue = reversed ? Byte.MAX_VALUE : Byte.MIN_VALUE;
+        } else {
+            dMissingValue = missingValue instanceof Number ? ((Number) missingValue).byteValue() : Byte.parseByte(missingValue.toString());
+        }
+
+        return new ByteValuesComparator(indexFieldData, dMissingValue, numHits);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java
new file mode 100644
index 00000000000..f63a6497267
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.index.fielddata.FloatValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class FloatValuesComparator extends FieldComparator<Float> {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final float missingValue;
+
+    protected final float[] values;
+    private float bottom;
+    private FloatValues readerValues;
+
+    public FloatValuesComparator(IndexNumericFieldData indexFieldData, float missingValue, int numHits) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+        this.values = new float[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final float v1 = values[slot1];
+        final float v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void setBottom(int slot) {
+        this.bottom = values[slot];
+    }
+
+    @Override
+    public int compareBottom(int doc) throws IOException {
+        float v2 = readerValues.getValueMissing(doc, missingValue);
+
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void copy(int slot, int doc) throws IOException {
+        values[slot] = readerValues.getValueMissing(doc, missingValue);
+    }
+
+    @Override
+    public FieldComparator<Float> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getFloatValues();
+        return this;
+    }
+
+    @Override
+    public Float value(int slot) {
+        return Float.valueOf(values[slot]);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Float valueObj) throws IOException {
+        final float value = valueObj.floatValue();
+        float docValue = readerValues.getValueMissing(doc, missingValue);
+        if (docValue < value) {
+            return -1;
+        } else if (docValue > value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
new file mode 100644
index 00000000000..50949de6ab2
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final Object missingValue;
+
+    public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.FLOAT;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+
+        float dMissingValue;
+        if (missingValue == null || "_last".equals(missingValue)) {
+            dMissingValue = reversed ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
+        } else if ("_first".equals(missingValue)) {
+            dMissingValue = reversed ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
+        } else {
+            dMissingValue = missingValue instanceof Number ? ((Number) missingValue).floatValue() : Float.parseFloat(missingValue.toString());
+        }
+
+        return new FloatValuesComparator(indexFieldData, dMissingValue, numHits);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparator.java
new file mode 100644
index 00000000000..064ad2888b7
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.IntValues;
+
+import java.io.IOException;
+
+/**
+ */
+public class IntValuesComparator extends FieldComparator<Integer> {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final int missingValue;
+
+    protected final int[] values;
+    private int bottom;
+    private IntValues readerValues;
+
+    public IntValuesComparator(IndexNumericFieldData indexFieldData, int missingValue, int numHits) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+        this.values = new int[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final int v1 = values[slot1];
+        final int v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void setBottom(int slot) {
+        this.bottom = values[slot];
+    }
+
+    @Override
+    public int compareBottom(int doc) throws IOException {
+        int v2 = readerValues.getValueMissing(doc, missingValue);
+
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void copy(int slot, int doc) throws IOException {
+        values[slot] = readerValues.getValueMissing(doc, missingValue);
+    }
+
+    @Override
+    public FieldComparator<Integer> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getIntValues();
+        return this;
+    }
+
+    @Override
+    public Integer value(int slot) {
+        return Integer.valueOf(values[slot]);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Integer valueObj) throws IOException {
+        final int value = valueObj.intValue();
+        int docValue = readerValues.getValueMissing(doc, missingValue);
+        if (docValue < value) {
+            return -1;
+        } else if (docValue > value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java
new file mode 100644
index 00000000000..bebbd479adf
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class IntValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final Object missingValue;
+
+    public IntValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.INT;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+
+        int dMissingValue;
+        if (missingValue == null || "_last".equals(missingValue)) {
+            dMissingValue = reversed ? Integer.MIN_VALUE : Integer.MAX_VALUE;
+        } else if ("_first".equals(missingValue)) {
+            dMissingValue = reversed ? Integer.MAX_VALUE : Integer.MIN_VALUE;
+        } else {
+            dMissingValue = missingValue instanceof Number ? ((Number) missingValue).intValue() : Integer.parseInt(missingValue.toString());
+        }
+
+        return new IntValuesComparator(indexFieldData, dMissingValue, numHits);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java
new file mode 100644
index 00000000000..abef68b1a07
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
+
+import java.io.IOException;
+
+/**
+ */
+public class LongValuesComparator extends FieldComparator<Long> {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final long missingValue;
+
+    protected final long[] values;
+    private long bottom;
+    private LongValues readerValues;
+
+    public LongValuesComparator(IndexNumericFieldData indexFieldData, long missingValue, int numHits) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+        this.values = new long[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final long v1 = values[slot1];
+        final long v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void setBottom(int slot) {
+        this.bottom = values[slot];
+    }
+
+    @Override
+    public int compareBottom(int doc) throws IOException {
+        long v2 = readerValues.getValueMissing(doc, missingValue);
+
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void copy(int slot, int doc) throws IOException {
+        values[slot] = readerValues.getValueMissing(doc, missingValue);
+    }
+
+    @Override
+    public FieldComparator<Long> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getLongValues();
+        return this;
+    }
+
+    @Override
+    public Long value(int slot) {
+        return Long.valueOf(values[slot]);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Long valueObj) throws IOException {
+        final long value = valueObj.longValue();
+        long docValue = readerValues.getValueMissing(doc, missingValue);
+        if (docValue < value) {
+            return -1;
+        } else if (docValue > value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
new file mode 100644
index 00000000000..05878f3b0d6
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final Object missingValue;
+
+    public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.LONG;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+
+        long dMissingValue;
+        if (missingValue == null || "_last".equals(missingValue)) {
+            dMissingValue = reversed ? Long.MIN_VALUE : Long.MAX_VALUE;
+        } else if ("_first".equals(missingValue)) {
+            dMissingValue = reversed ? Long.MAX_VALUE : Long.MIN_VALUE;
+        } else {
+            dMissingValue = missingValue instanceof Number ? ((Number) missingValue).longValue() : Long.parseLong(missingValue.toString());
+        }
+
+        return new LongValuesComparator(indexFieldData, dMissingValue, numHits);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparator.java
new file mode 100644
index 00000000000..4b85071f5a4
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.ShortValues;
+
+import java.io.IOException;
+
+/**
+ */
+public class ShortValuesComparator extends FieldComparator<Short> {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final short missingValue;
+
+    protected final short[] values;
+    private short bottom;
+    private ShortValues readerValues;
+
+    public ShortValuesComparator(IndexNumericFieldData indexFieldData, short missingValue, int numHits) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+        this.values = new short[numHits];
+    }
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        final short v1 = values[slot1];
+        final short v2 = values[slot2];
+        if (v1 > v2) {
+            return 1;
+        } else if (v1 < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void setBottom(int slot) {
+        this.bottom = values[slot];
+    }
+
+    @Override
+    public int compareBottom(int doc) throws IOException {
+        short v2 = readerValues.getValueMissing(doc, missingValue);
+
+        if (bottom > v2) {
+            return 1;
+        } else if (bottom < v2) {
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void copy(int slot, int doc) throws IOException {
+        values[slot] = readerValues.getValueMissing(doc, missingValue);
+    }
+
+    @Override
+    public FieldComparator<Short> setNextReader(AtomicReaderContext context) throws IOException {
+        this.readerValues = indexFieldData.load(context).getShortValues();
+        return this;
+    }
+
+    @Override
+    public Short value(int slot) {
+        return Short.valueOf(values[slot]);
+    }
+
+    @Override
+    public int compareDocToValue(int doc, Short valueObj) throws IOException {
+        final short value = valueObj.shortValue();
+        short docValue = readerValues.getValueMissing(doc, missingValue);
+        if (docValue < value) {
+            return -1;
+        } else if (docValue > value) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparatorSource.java
new file mode 100644
index 00000000000..0b7a883b251
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/ShortValuesComparatorSource.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.fieldcomparator;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+
+import java.io.IOException;
+
+/**
+ */
+public class ShortValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
+
+    private final IndexNumericFieldData indexFieldData;
+    private final Object missingValue;
+
+    public ShortValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
+        this.indexFieldData = indexFieldData;
+        this.missingValue = missingValue;
+    }
+
+    @Override
+    public SortField.Type reducedType() {
+        return SortField.Type.INT;
+    }
+
+    @Override
+    public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+        assert fieldname.equals(indexFieldData.getFieldNames().indexName());
+
+        short dMissingValue;
+        if (missingValue == null || "_last".equals(missingValue)) {
+            dMissingValue = reversed ? Short.MIN_VALUE : Short.MAX_VALUE;
+        } else if ("_first".equals(missingValue)) {
+            dMissingValue = reversed ? Short.MAX_VALUE : Short.MIN_VALUE;
+        } else {
+            dMissingValue = missingValue instanceof Number ? ((Number) missingValue).shortValue() : Short.parseShort(missingValue.toString());
+        }
+
+        return new ShortValuesComparator(indexFieldData, dMissingValue, numHits);
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
index c4694089a84..dd6fdb76a24 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.fieldcomparator.ByteValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -152,6 +152,6 @@ public class ByteArrayIndexFieldData extends AbstractIndexFieldData<ByteArrayAto
 
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
-        return new DoubleValuesComparatorSource(this, missingValue);
+        return new ByteValuesComparatorSource(this, missingValue);
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
index 6a921b8a7c1..2471c1af68f 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -152,6 +152,6 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
 
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
-        return new DoubleValuesComparatorSource(this, missingValue);
+        return new FloatValuesComparatorSource(this, missingValue);
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
index f42baad69a0..94bf1d429dc 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.fieldcomparator.IntValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -152,6 +152,6 @@ public class IntArrayIndexFieldData extends AbstractIndexFieldData<IntArrayAtomi
 
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
-        return new DoubleValuesComparatorSource(this, missingValue);
+        return new IntValuesComparatorSource(this, missingValue);
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
index b38ead65e84..dc109d62129 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -152,6 +152,6 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
 
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
-        return new DoubleValuesComparatorSource(this, missingValue);
+        return new LongValuesComparatorSource(this, missingValue);
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
index d1f7c107ac4..33e869bf49c 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
-import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
+import org.elasticsearch.index.fielddata.fieldcomparator.ShortValuesComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -152,6 +152,6 @@ public class ShortArrayIndexFieldData extends AbstractIndexFieldData<ShortArrayA
 
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
-        return new DoubleValuesComparatorSource(this, missingValue);
+        return new ShortValuesComparatorSource(this, missingValue);
     }
 }

From 5b7173fc35cf2dbcfccb346cbb0271c07e9674a6 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Mon, 21 Jan 2013 10:50:25 +0100
Subject: [PATCH 24/35] move sorting to work with new field data

---
 .../org/elasticsearch/common/lucene/Lucene.java  |  4 ++--
 .../index/fielddata/IndexFieldDataService.java   |  2 +-
 .../fielddata/plain/ByteArrayIndexFieldData.java |  2 +-
 .../plain/ConcreteBytesRefAtomicFieldData.java   |  2 +-
 .../plain/DoubleArrayIndexFieldData.java         |  2 +-
 .../plain/FloatArrayIndexFieldData.java          |  2 +-
 .../plain/GeoPointDoubleArrayIndexFieldData.java |  2 +-
 .../fielddata/plain/IntArrayIndexFieldData.java  |  2 +-
 .../fielddata/plain/LongArrayIndexFieldData.java |  2 +-
 ...dData.java => PagedBytesAtomicFieldData.java} |  4 ++--
 ...ldData.java => PagesBytesIndexFieldData.java} | 16 ++++++++--------
 .../plain/ShortArrayIndexFieldData.java          |  2 +-
 .../search/sort/SortParseElement.java            |  2 +-
 .../integration/search/sort/SimpleSortTests.java |  9 ++++++---
 ....java => PagedBytesStringFieldDataTests.java} |  4 ++--
 .../index/fielddata/StringFieldDataTests.java    | 10 ++++++++++
 16 files changed, 40 insertions(+), 27 deletions(-)
 rename src/main/java/org/elasticsearch/index/fielddata/plain/{PackedBytesAtomicFieldData.java => PagedBytesAtomicFieldData.java} (99%)
 rename src/main/java/org/elasticsearch/index/fielddata/plain/{PackedBytesIndexFieldData.java => PagesBytesIndexFieldData.java} (87%)
 rename src/test/java/org/elasticsearch/test/unit/index/fielddata/{PackedBytesStringFieldDataTests.java => PagedBytesStringFieldDataTests.java} (86%)

diff --git a/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 5fabf1c00b9..b0bd62b6c8d 100644
--- a/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -32,7 +32,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 
 import java.io.IOException;
 import java.lang.reflect.Field;
@@ -215,7 +215,7 @@ public class Lucene {
                     out.writeString(sortField.getField());
                 }
                 if (sortField.getComparatorSource() != null) {
-                    writeSortType(out, ((FieldDataType.ExtendedFieldComparatorSource) sortField.getComparatorSource()).reducedType());
+                    writeSortType(out, ((IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource()).reducedType());
                 } else {
                     writeSortType(out, sortField.getType());
                 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index 2cb4d966f9b..8773b159cd9 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -56,7 +56,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
 
         buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
-                .put(Tuple.tuple("string", "packed_bytes"), new PackedBytesIndexFieldData.Builder())
+                .put(Tuple.tuple("string", "paged_bytes"), new PagesBytesIndexFieldData.Builder())
                 .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("byte", "array"), new ByteArrayIndexFieldData.Builder())
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
index dd6fdb76a24..3fcb6ab0cdb 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class ByteArrayIndexFieldData extends AbstractIndexFieldData<ByteArrayAto
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new ByteArrayAtomicFieldData.Single(new byte[0], 0);
+            return new ByteArrayAtomicFieldData.SingleFixedSet(new byte[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
index ea036083744..127e661f64c 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
@@ -131,7 +131,7 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
             } else {
                 ret.bytes = value.bytes;
                 ret.offset = value.offset;
-                ret.length = value.offset;
+                ret.length = value.length;
             }
             return ret;
         }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
index b6d474dd68a..d178dffc482 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new DoubleArrayAtomicFieldData.Single(new double[0], 0);
+            return new DoubleArrayAtomicFieldData.SingleFixedSet(new double[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
index 2471c1af68f..3615e560fa1 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new FloatArrayAtomicFieldData.Single(new float[0], 0);
+            return new FloatArrayAtomicFieldData.SingleFixedSet(new float[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
index 11a89ee5bb4..c1ccd1aee31 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
@@ -77,7 +77,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<Ge
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new GeoPointDoubleArrayAtomicFieldData.Single(new double[0], new double[0], 0);
+            return new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(new double[1], new double[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
index 94bf1d429dc..556ab37f75f 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class IntArrayIndexFieldData extends AbstractIndexFieldData<IntArrayAtomi
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new IntArrayAtomicFieldData.Single(new int[0], 0);
+            return new IntArrayAtomicFieldData.SingleFixedSet(new int[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
index dc109d62129..28697712772 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new LongArrayAtomicFieldData.Single(new long[0], 0);
+            return new LongArrayAtomicFieldData.SingleFixedSet(new long[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
similarity index 99%
rename from src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesAtomicFieldData.java
rename to src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
index 0e251617e25..ccdf864fbb2 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
@@ -32,7 +32,7 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
 
 /**
  */
-public class PackedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
+public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
 
     // 0 ordinal in values means no value (its null)
     private final PagedBytes.Reader bytes;
@@ -42,7 +42,7 @@ public class PackedBytesAtomicFieldData implements AtomicOrdinalFieldData<Script
     private int[] hashes;
     private long size = -1;
 
-    public PackedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals ordinals) {
+    public PagedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals ordinals) {
         this.bytes = bytes;
         this.termOrdToBytesOffset = termOrdToBytesOffset;
         this.ordinals = ordinals;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagesBytesIndexFieldData.java
similarity index 87%
rename from src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesIndexFieldData.java
rename to src/main/java/org/elasticsearch/index/fielddata/plain/PagesBytesIndexFieldData.java
index fb70a1f000f..fb6ef4addfa 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedBytesIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagesBytesIndexFieldData.java
@@ -41,17 +41,17 @@ import java.util.ArrayList;
 
 /**
  */
-public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedBytesAtomicFieldData> implements IndexOrdinalFieldData<PackedBytesAtomicFieldData> {
+public class PagesBytesIndexFieldData extends AbstractIndexFieldData<PagedBytesAtomicFieldData> implements IndexOrdinalFieldData<PagedBytesAtomicFieldData> {
 
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
         public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
-            return new PackedBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
+            return new PagesBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
         }
     }
 
-    public PackedBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+    public PagesBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
         super(index, indexSettings, fieldNames, fieldDataType, cache);
     }
 
@@ -61,7 +61,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
     }
 
     @Override
-    public PackedBytesAtomicFieldData load(AtomicReaderContext context) {
+    public PagedBytesAtomicFieldData load(AtomicReaderContext context) {
         try {
             return cache.load(context, this);
         } catch (Throwable e) {
@@ -74,7 +74,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
     }
 
     @Override
-    public PackedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+    public PagedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
 
         Terms terms = reader.terms(getFieldNames().indexName());
@@ -83,7 +83,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
             // 0 is reserved for "unset"
             bytes.copyUsingLengthPrefix(new BytesRef());
             GrowableWriter termOrdToBytesOffset = new GrowableWriter(1, 2, PackedInts.FASTEST);
-            return new PackedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
+            return new PagedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
         }
 
         final PagedBytes bytes = new PagedBytes(15);
@@ -174,13 +174,13 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
         PackedInts.Reader termOrdToBytesOffsetReader = termOrdToBytesOffset.getMutable();
 
         if (ordinals.size() == 1) {
-            return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new SingleArrayOrdinals(ordinals.get(0), termOrd));
+            return new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new SingleArrayOrdinals(ordinals.get(0), termOrd));
         } else {
             int[][] nativeOrdinals = new int[ordinals.size()][];
             for (int i = 0; i < nativeOrdinals.length; i++) {
                 nativeOrdinals[i] = ordinals.get(i);
             }
-            return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+            return new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
         }
 
     }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
index 33e869bf49c..518b4c8a5f0 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class ShortArrayIndexFieldData extends AbstractIndexFieldData<ShortArrayA
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new ShortArrayAtomicFieldData.Single(new short[0], 0);
+            return new ShortArrayAtomicFieldData.SingleFixedSet(new short[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java
index d9a1b27622d..0ca40e483f8 100644
--- a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java
+++ b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java
@@ -163,7 +163,7 @@ public class SortParseElement implements SearchParseElement {
                 }
                 throw new SearchParseException(context, "No mapping found for [" + fieldName + "] in order to sort on");
             }
-            sortFields.add(new SortField(fieldMapper.names().indexName(), fieldMapper.fieldDataType().newFieldComparatorSource(context.fieldDataCache(), missing), reverse));
+            sortFields.add(new SortField(fieldMapper.names().indexName(), context.fieldData().getForField(fieldMapper).comparatorSource(missing), reverse));
         }
     }
 }
diff --git a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java
index 8fa893252d8..01e019838c1 100644
--- a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java
+++ b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java
@@ -531,22 +531,24 @@ public class SimpleSortTests extends AbstractNodesTests {
 
         client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet();
 
-        logger.info("--> sort with no missing");
+        logger.info("--> sort with no missing (same as missing _last)");
         SearchResponse searchResponse = client.prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC))
                 .execute().actionGet();
+        assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
 
         assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
         assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
-        assertThat(searchResponse.hits().getAt(1).id(), equalTo("2"));
-        assertThat(searchResponse.hits().getAt(2).id(), equalTo("3"));
+        assertThat(searchResponse.hits().getAt(1).id(), equalTo("3"));
+        assertThat(searchResponse.hits().getAt(2).id(), equalTo("2"));
 
         logger.info("--> sort with missing _last");
         searchResponse = client.prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last"))
                 .execute().actionGet();
+        assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
 
         assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
         assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
@@ -558,6 +560,7 @@ public class SimpleSortTests extends AbstractNodesTests {
                 .setQuery(matchAllQuery())
                 .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first"))
                 .execute().actionGet();
+        assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
 
         assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
         assertThat(searchResponse.hits().getAt(0).id(), equalTo("2"));
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/PackedBytesStringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/PagedBytesStringFieldDataTests.java
similarity index 86%
rename from src/test/java/org/elasticsearch/test/unit/index/fielddata/PackedBytesStringFieldDataTests.java
rename to src/test/java/org/elasticsearch/test/unit/index/fielddata/PagedBytesStringFieldDataTests.java
index 8dfdd1030f1..f95e92e389a 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/PackedBytesStringFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/PagedBytesStringFieldDataTests.java
@@ -26,10 +26,10 @@ import org.testng.annotations.Test;
 /**
  */
 @Test
-public class PackedBytesStringFieldDataTests extends StringFieldDataTests {
+public class PagedBytesStringFieldDataTests extends StringFieldDataTests {
 
     @Override
     protected FieldDataType getFieldDataType() {
-        return new FieldDataType("string", "packed_bytes", ImmutableMap.<String, String>of());
+        return new FieldDataType("string", "paged_bytes", ImmutableMap.<String, String>of());
     }
 }
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
index 439329eaec9..7d8cc181434 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
@@ -56,6 +56,13 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         return "4";
     }
 
+    private String toString(Object value) {
+        if (value instanceof BytesRef) {
+            return ((BytesRef) value).utf8ToString();
+        }
+        return value.toString();
+    }
+
     protected void fillSingleValueAllSet() throws Exception {
         Document d = new Document();
         d.add(new StringField("_id", "1", Field.Store.NO));
@@ -218,8 +225,11 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
                 new Sort(new SortField("value", indexFieldData.comparatorSource(null))));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two()));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
+        assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                 new Sort(new SortField("value", indexFieldData.comparatorSource(null), true)));

From f189a832c5067fa4feb4b1b0567c4531ca32f661 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Mon, 21 Jan 2013 11:45:09 +0100
Subject: [PATCH 25/35] grr pages -> paged

---
 .../index/fielddata/IndexFieldDataService.java              | 2 +-
 ...tesIndexFieldData.java => PagedBytesIndexFieldData.java} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename src/main/java/org/elasticsearch/index/fielddata/plain/{PagesBytesIndexFieldData.java => PagedBytesIndexFieldData.java} (97%)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index 8773b159cd9..b97a0f3ca02 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -56,7 +56,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
 
         buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
-                .put(Tuple.tuple("string", "paged_bytes"), new PagesBytesIndexFieldData.Builder())
+                .put(Tuple.tuple("string", "paged_bytes"), new PagedBytesIndexFieldData.Builder())
                 .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("byte", "array"), new ByteArrayIndexFieldData.Builder())
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PagesBytesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
similarity index 97%
rename from src/main/java/org/elasticsearch/index/fielddata/plain/PagesBytesIndexFieldData.java
rename to src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
index fb6ef4addfa..def03de6abb 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PagesBytesIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
@@ -41,17 +41,17 @@ import java.util.ArrayList;
 
 /**
  */
-public class PagesBytesIndexFieldData extends AbstractIndexFieldData<PagedBytesAtomicFieldData> implements IndexOrdinalFieldData<PagedBytesAtomicFieldData> {
+public class PagedBytesIndexFieldData extends AbstractIndexFieldData<PagedBytesAtomicFieldData> implements IndexOrdinalFieldData<PagedBytesAtomicFieldData> {
 
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
         public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
-            return new PagesBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
+            return new PagedBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
         }
     }
 
-    public PagesBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+    public PagedBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
         super(index, indexSettings, fieldNames, fieldDataType, cache);
     }
 

From e5b651321fcd31424c2be6be5962deaa2890d3a4 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Mon, 21 Jan 2013 16:29:16 +0100
Subject: [PATCH 26/35] remove some safe methods because of the new makeSafe
 method usage

---
 .../index/fielddata/BytesValues.java          |  74 ---------
 .../index/fielddata/HashedBytesValues.java    | 136 +---------------
 .../ConcreteBytesRefAtomicFieldData.java      |  97 -----------
 .../plain/PagedBytesAtomicFieldData.java      | 152 ------------------
 .../index/fielddata/StringFieldDataTests.java | 124 --------------
 5 files changed, 3 insertions(+), 580 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
index 20d1502d632..b44faf79d09 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
@@ -55,11 +55,6 @@ public interface BytesValues {
      */
     BytesRef getValueScratch(int docId, BytesRef ret);
 
-    /**
-     * Returns a bytes value for a docId. The content is guaranteed not to be shared.
-     */
-    BytesRef getValueSafe(int docId);
-
     /**
      * Returns an array wrapping all the bytes values for a doc. The content is guaranteed not to be shared.
      */
@@ -70,21 +65,11 @@ public interface BytesValues {
      */
     Iter getIter(int docId);
 
-    /**
-     * Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
-     */
-    Iter getIterSafe(int docId);
-
     /**
      * Go over all the possible values in their BytesRef format for a specific doc.
      */
     void forEachValueInDoc(int docId, ValueInDocProc proc);
 
-    /**
-     * Go over all the possible values in their BytesRef format for a specific doc.
-     */
-    void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
-
     public static interface ValueInDocProc {
         void onValue(int docId, BytesRef value);
 
@@ -144,9 +129,7 @@ public interface BytesValues {
         protected final BytesRef scratch = new BytesRef();
         private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
         private final ValueIter valueIter = new ValueIter();
-        private final SafeValueIter safeValueIter = new SafeValueIter();
         private final Proc proc = new Proc();
-        private final SafeProc safeProc = new SafeProc();
 
         public StringBased(StringValues values) {
             this.values = values;
@@ -187,13 +170,6 @@ public interface BytesValues {
             return ret;
         }
 
-        @Override
-        public BytesRef getValueSafe(int docId) {
-            String value = values.getValue(docId);
-            if (value == null) return null;
-            return new BytesRef(value);
-        }
-
         @Override
         public BytesRefArrayRef getValues(int docId) {
             StringArrayRef arrayRef = values.getValues(docId);
@@ -214,21 +190,11 @@ public interface BytesValues {
             return valueIter.reset(values.getIter(docId));
         }
 
-        @Override
-        public Iter getIterSafe(int docId) {
-            return safeValueIter.reset(values.getIter(docId));
-        }
-
         @Override
         public void forEachValueInDoc(int docId, ValueInDocProc proc) {
             values.forEachValueInDoc(docId, this.proc.reset(proc));
         }
 
-        @Override
-        public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
-        }
-
         static class ValueIter implements Iter {
 
             private final BytesRef scratch = new BytesRef();
@@ -251,26 +217,6 @@ public interface BytesValues {
             }
         }
 
-        static class SafeValueIter implements Iter {
-
-            private StringValues.Iter iter;
-
-            public SafeValueIter reset(StringValues.Iter iter) {
-                this.iter = iter;
-                return this;
-            }
-
-            @Override
-            public boolean hasNext() {
-                return iter.hasNext();
-            }
-
-            @Override
-            public BytesRef next() {
-                return new BytesRef(iter.next());
-            }
-        }
-
         static class Proc implements StringValues.ValueInDocProc {
 
             private final BytesRef scratch = new BytesRef();
@@ -292,25 +238,5 @@ public interface BytesValues {
                 proc.onMissing(docId);
             }
         }
-
-        static class SafeProc implements StringValues.ValueInDocProc {
-
-            private BytesValues.ValueInDocProc proc;
-
-            public SafeProc reset(BytesValues.ValueInDocProc proc) {
-                this.proc = proc;
-                return this;
-            }
-
-            @Override
-            public void onValue(int docId, String value) {
-                proc.onValue(docId, new BytesRef(value));
-            }
-
-            @Override
-            public void onMissing(int docId) {
-                proc.onMissing(docId);
-            }
-        }
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
index 2fe6f6da61b..f3479b900f2 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
@@ -43,35 +43,22 @@ public interface HashedBytesValues {
     HashedBytesRef makeSafe(HashedBytesRef bytes);
 
     /**
-     * Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
+     * Returns a bytes value for a docId. Note, the content of it might be shared across invocation,
+     * call {@link #makeSafe(org.elasticsearch.common.lucene.HashedBytesRef)} to converts it to a "safe"
+     * option (if needed).
      */
     HashedBytesRef getValue(int docId);
 
-    /**
-     * Returns a bytes value for a docId. The content is guaranteed not to be shared.
-     */
-    HashedBytesRef getValueSafe(int docId);
-
     /**
      * Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
      */
     Iter getIter(int docId);
 
-    /**
-     * Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
-     */
-    Iter getIterSafe(int docId);
-
     /**
      * Go over all the possible values in their BytesRef format for a specific doc.
      */
     void forEachValueInDoc(int docId, ValueInDocProc proc);
 
-    /**
-     * Go over all the possible values in their BytesRef format for a specific doc.
-     */
-    void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
-
     public static interface ValueInDocProc {
         void onValue(int docId, HashedBytesRef value);
 
@@ -133,9 +120,7 @@ public interface HashedBytesValues {
 
         protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
         private final ValueIter valueIter = new ValueIter();
-        private final SafeValueIter safeValueIter = new SafeValueIter();
         private final Proc proc = new Proc();
-        private final SafeProc safeProc = new SafeProc();
 
         public BytesBased(BytesValues values) {
             this.values = values;
@@ -164,31 +149,16 @@ public interface HashedBytesValues {
             return scratch.resetHashCode();
         }
 
-        @Override
-        public HashedBytesRef getValueSafe(int docId) {
-            return new HashedBytesRef(values.getValueSafe(docId));
-        }
-
         @Override
         public Iter getIter(int docId) {
             return valueIter.reset(values.getIter(docId));
         }
 
-        @Override
-        public Iter getIterSafe(int docId) {
-            return safeValueIter.reset(values.getIterSafe(docId));
-        }
-
         @Override
         public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
             values.forEachValueInDoc(docId, this.proc.reset(proc));
         }
 
-        @Override
-        public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
-            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
-        }
-
         static class ValueIter implements Iter {
 
             private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
@@ -211,26 +181,6 @@ public interface HashedBytesValues {
             }
         }
 
-        static class SafeValueIter implements Iter {
-
-            private BytesValues.Iter iter;
-
-            public SafeValueIter reset(BytesValues.Iter iter) {
-                this.iter = iter;
-                return this;
-            }
-
-            @Override
-            public boolean hasNext() {
-                return iter.hasNext();
-            }
-
-            @Override
-            public HashedBytesRef next() {
-                return new HashedBytesRef(iter.next());
-            }
-        }
-
         static class Proc implements BytesValues.ValueInDocProc {
 
             private final HashedBytesRef scratch = new HashedBytesRef();
@@ -252,27 +202,6 @@ public interface HashedBytesValues {
                 proc.onMissing(docId);
             }
         }
-
-        static class SafeProc implements BytesValues.ValueInDocProc {
-
-            private ValueInDocProc proc;
-
-            public SafeProc reset(ValueInDocProc proc) {
-                this.proc = proc;
-                return this;
-            }
-
-            @Override
-            public void onValue(int docId, BytesRef value) {
-                proc.onValue(docId, new HashedBytesRef(value));
-            }
-
-            @Override
-            public void onMissing(int docId) {
-                proc.onMissing(docId);
-            }
-        }
-
     }
 
     static class StringBased implements HashedBytesValues {
@@ -281,9 +210,7 @@ public interface HashedBytesValues {
 
         protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
         private final ValueIter valueIter = new ValueIter();
-        private final SafeValueIter safeValueIter = new SafeValueIter();
         private final Proc proc = new Proc();
-        private final SafeProc safeProc = new SafeProc();
 
         public StringBased(StringValues values) {
             this.values = values;
@@ -313,33 +240,16 @@ public interface HashedBytesValues {
             return scratch.resetHashCode();
         }
 
-        @Override
-        public HashedBytesRef getValueSafe(int docId) {
-            String value = values.getValue(docId);
-            if (value == null) return null;
-            return new HashedBytesRef(new BytesRef(values.getValue(docId)));
-        }
-
         @Override
         public Iter getIter(int docId) {
             return valueIter.reset(values.getIter(docId));
         }
 
-        @Override
-        public Iter getIterSafe(int docId) {
-            return safeValueIter.reset(values.getIter(docId));
-        }
-
         @Override
         public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
             values.forEachValueInDoc(docId, this.proc.reset(proc));
         }
 
-        @Override
-        public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
-            values.forEachValueInDoc(docId, this.safeProc.reset(proc));
-        }
-
         static class ValueIter implements Iter {
 
             private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
@@ -362,26 +272,6 @@ public interface HashedBytesValues {
             }
         }
 
-        static class SafeValueIter implements Iter {
-
-            private StringValues.Iter iter;
-
-            public SafeValueIter reset(StringValues.Iter iter) {
-                this.iter = iter;
-                return this;
-            }
-
-            @Override
-            public boolean hasNext() {
-                return iter.hasNext();
-            }
-
-            @Override
-            public HashedBytesRef next() {
-                return new HashedBytesRef(new BytesRef(iter.next()));
-            }
-        }
-
         static class Proc implements StringValues.ValueInDocProc {
 
             private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
@@ -403,25 +293,5 @@ public interface HashedBytesValues {
                 proc.onMissing(docId);
             }
         }
-
-        static class SafeProc implements StringValues.ValueInDocProc {
-
-            private ValueInDocProc proc;
-
-            public SafeProc reset(ValueInDocProc proc) {
-                this.proc = proc;
-                return this;
-            }
-
-            @Override
-            public void onValue(int docId, String value) {
-                proc.onValue(docId, new HashedBytesRef(new BytesRef(value)));
-            }
-
-            @Override
-            public void onMissing(int docId) {
-                proc.onMissing(docId);
-            }
-        }
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
index 127e661f64c..fcafcba97bd 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
@@ -170,16 +170,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
             return ret;
         }
 
-        @Override
-        public BytesRef getValueSafe(int docId) {
-            return values[ordinals.getOrd(docId)];
-        }
-
-        @Override
-        public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-            forEachValueInDoc(docId, proc);
-        }
-
         static class Single extends BytesValues {
 
             private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
@@ -209,13 +199,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
                 return iter.reset(values[ord]);
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                int ord = ordinals.getOrd(docId);
-                if (ord == 0) return Iter.Empty.INSTANCE;
-                return iter.reset(values[ord]);
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 int ord = ordinals.getOrd(docId);
@@ -260,11 +243,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
                 return iter.reset(ordinals.getIter(docId));
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                return iter.reset(ordinals.getIter(docId));
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 Ordinals.Docs.Iter iter = ordinals.getIter(docId);
@@ -357,13 +335,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
             return scratch.reset(values[ord], hashes[ord]);
         }
 
-        @Override
-        public HashedBytesRef getValueSafe(int docId) {
-            int ord = ordinals.getOrd(docId);
-            if (ord == 0) return null;
-            return new HashedBytesRef(values[ord], hashes[ord]);
-        }
-
         static class Single extends HashedBytesValues {
 
             private final Iter.Single iter = new Iter.Single();
@@ -384,13 +355,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
                 return iter.reset(scratch.reset(values[ord], hashes[ord]));
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                int ord = ordinals.getOrd(docId);
-                if (ord == 0) return Iter.Empty.INSTANCE;
-                return iter.reset(new HashedBytesRef(values[ord], hashes[ord]));
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 int ord = ordinals.getOrd(docId);
@@ -400,27 +364,15 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
                     proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
                 }
             }
-
-            @Override
-            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-                int ord = ordinals.getOrd(docId);
-                if (ord == 0) {
-                    proc.onMissing(docId);
-                } else {
-                    proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
-                }
-            }
         }
 
         static class Multi extends HashedBytesValues {
 
             private final ValuesIter iter;
-            private final SafeValuesIter safeIter;
 
             Multi(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
                 super(values, hashes, ordinals);
                 this.iter = new ValuesIter(values, hashes);
-                this.safeIter = new SafeValuesIter(values, hashes);
             }
 
             @Override
@@ -433,11 +385,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
                 return iter.reset(ordinals.getIter(docId));
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                return safeIter.reset(ordinals.getIter(docId));
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 Ordinals.Docs.Iter iter = ordinals.getIter(docId);
@@ -451,19 +398,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
                 } while ((ord = iter.next()) != 0);
             }
 
-            @Override
-            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
-                int ord = iter.next();
-                if (ord == 0) {
-                    proc.onMissing(docId);
-                    return;
-                }
-                do {
-                    proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
-                } while ((ord = iter.next()) != 0);
-            }
-
             static class ValuesIter implements Iter {
 
                 private final BytesRef[] values;
@@ -496,37 +430,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
                     return value;
                 }
             }
-
-            static class SafeValuesIter implements Iter {
-
-                private final BytesRef[] values;
-                private final int[] hashes;
-                private Ordinals.Docs.Iter ordsIter;
-                private int ord;
-
-                SafeValuesIter(BytesRef[] values, int[] hashes) {
-                    this.values = values;
-                    this.hashes = hashes;
-                }
-
-                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
-                    this.ordsIter = ordsIter;
-                    this.ord = ordsIter.next();
-                    return this;
-                }
-
-                @Override
-                public boolean hasNext() {
-                    return ord != 0;
-                }
-
-                @Override
-                public HashedBytesRef next() {
-                    HashedBytesRef value = new HashedBytesRef(values[ord], hashes[ord]);
-                    ord = ordsIter.next();
-                    return value;
-                }
-            }
         }
     }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
index ccdf864fbb2..79bdb7da73e 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
@@ -165,13 +165,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
             return bytes.fill(ret, termOrdToBytesOffset.get(ordinals.getOrd(docId)));
         }
 
-        @Override
-        public BytesRef getValueSafe(int docId) {
-            int ord = ordinals.getOrd(docId);
-            if (ord == 0) return null;
-            return bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord));
-        }
-
         static class Single extends BytesValues {
 
             private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
@@ -201,13 +194,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                 return iter.reset(bytes.fill(scratch, termOrdToBytesOffset.get(ord)));
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                int ord = ordinals.getOrd(docId);
-                if (ord == 0) return Iter.Empty.INSTANCE;
-                return iter.reset(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)));
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 int ord = ordinals.getOrd(docId);
@@ -217,28 +203,16 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                     proc.onValue(docId, bytes.fill(scratch, termOrdToBytesOffset.get(ord)));
                 }
             }
-
-            @Override
-            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-                int ord = ordinals.getOrd(docId);
-                if (ord == 0) {
-                    proc.onMissing(docId);
-                } else {
-                    proc.onValue(docId, bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)));
-                }
-            }
         }
 
         static class Multi extends BytesValues {
 
             private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[10], 0);
             private final ValuesIter iter;
-            private final SafeValuesIter safeIter;
 
             Multi(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
                 super(bytes, termOrdToBytesOffset, ordinals);
                 this.iter = new ValuesIter(bytes, termOrdToBytesOffset);
-                this.safeIter = new SafeValuesIter(bytes, termOrdToBytesOffset);
             }
 
             @Override
@@ -264,11 +238,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                 return iter.reset(ordinals.getIter(docId));
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                return safeIter.reset(ordinals.getIter(docId));
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 Ordinals.Docs.Iter iter = ordinals.getIter(docId);
@@ -282,19 +251,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                 } while ((ord = iter.next()) != 0);
             }
 
-            @Override
-            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
-                int ord = iter.next();
-                if (ord == 0) {
-                    proc.onMissing(docId);
-                    return;
-                }
-                do {
-                    proc.onValue(docId, bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)));
-                } while ((ord = iter.next()) != 0);
-            }
-
             static class ValuesIter implements Iter {
 
                 private final PagedBytes.Reader bytes;
@@ -326,37 +282,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                     return value;
                 }
             }
-
-            static class SafeValuesIter implements Iter {
-
-                private final PagedBytes.Reader bytes;
-                private final PackedInts.Reader termOrdToBytesOffset;
-                private Ordinals.Docs.Iter ordsIter;
-                private int ord;
-
-                SafeValuesIter(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset) {
-                    this.bytes = bytes;
-                    this.termOrdToBytesOffset = termOrdToBytesOffset;
-                }
-
-                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
-                    this.ordsIter = ordsIter;
-                    this.ord = ordsIter.next();
-                    return this;
-                }
-
-                @Override
-                public boolean hasNext() {
-                    return ord != 0;
-                }
-
-                @Override
-                public BytesRef next() {
-                    BytesRef value = bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord));
-                    ord = ordsIter.next();
-                    return value;
-                }
-            }
         }
     }
 
@@ -409,13 +334,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
             return scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]);
         }
 
-        @Override
-        public HashedBytesRef getValueSafe(int docId) {
-            int ord = ordinals.getOrd(docId);
-            if (ord == 0) return null;
-            return new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]);
-        }
-
         static class Single extends HashedBytesValues {
 
             private final Iter.Single iter = new Iter.Single();
@@ -436,13 +354,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                 return iter.reset(scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]));
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                int ord = ordinals.getOrd(docId);
-                if (ord == 0) return Iter.Empty.INSTANCE;
-                return iter.reset(new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]));
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 int ord = ordinals.getOrd(docId);
@@ -452,27 +363,15 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                     proc.onValue(docId, scratch.reset(bytes.fill(scratch1, termOrdToBytesOffset.get(ord)), hashes[ord]));
                 }
             }
-
-            @Override
-            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-                int ord = ordinals.getOrd(docId);
-                if (ord == 0) {
-                    proc.onMissing(docId);
-                } else {
-                    proc.onValue(docId, new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]));
-                }
-            }
         }
 
         static class Multi extends HashedBytesValues {
 
             private final ValuesIter iter;
-            private final SafeValuesIter safeIter;
 
             Multi(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, int[] hashes, Ordinals.Docs ordinals) {
                 super(bytes, termOrdToBytesOffset, hashes, ordinals);
                 this.iter = new ValuesIter(bytes, termOrdToBytesOffset, hashes);
-                this.safeIter = new SafeValuesIter(bytes, termOrdToBytesOffset, hashes);
             }
 
             @Override
@@ -485,11 +384,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                 return iter.reset(ordinals.getIter(docId));
             }
 
-            @Override
-            public Iter getIterSafe(int docId) {
-                return safeIter.reset(ordinals.getIter(docId));
-            }
-
             @Override
             public void forEachValueInDoc(int docId, ValueInDocProc proc) {
                 Ordinals.Docs.Iter iter = ordinals.getIter(docId);
@@ -503,19 +397,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                 } while ((ord = iter.next()) != 0);
             }
 
-            @Override
-            public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
-                Ordinals.Docs.Iter iter = ordinals.getIter(docId);
-                int ord = iter.next();
-                if (ord == 0) {
-                    proc.onMissing(docId);
-                    return;
-                }
-                do {
-                    proc.onValue(docId, new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]));
-                } while ((ord = iter.next()) != 0);
-            }
-
             static class ValuesIter implements Iter {
 
                 private final PagedBytes.Reader bytes;
@@ -551,39 +432,6 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
                     return value;
                 }
             }
-
-            static class SafeValuesIter implements Iter {
-
-                private final PagedBytes.Reader bytes;
-                private final PackedInts.Reader termOrdToBytesOffset;
-                private final int[] hashes;
-                private Ordinals.Docs.Iter ordsIter;
-                private int ord;
-
-                SafeValuesIter(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, int[] hashes) {
-                    this.bytes = bytes;
-                    this.termOrdToBytesOffset = termOrdToBytesOffset;
-                    this.hashes = hashes;
-                }
-
-                public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
-                    this.ordsIter = ordsIter;
-                    this.ord = ordsIter.next();
-                    return this;
-                }
-
-                @Override
-                public boolean hasNext() {
-                    return ord != 0;
-                }
-
-                @Override
-                public HashedBytesRef next() {
-                    HashedBytesRef value = new HashedBytesRef(bytes.fill(new BytesRef(), termOrdToBytesOffset.get(ord)), hashes[ord]);
-                    ord = ordsIter.next();
-                    return value;
-                }
-            }
         }
     }
 
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
index 7d8cc181434..799458874a3 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
@@ -101,10 +101,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
         assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
 
-        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
-        assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
-        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
-
         BytesRef bytesRef = new BytesRef();
         assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
         assertThat(bytesRef, equalTo(new BytesRef(two())));
@@ -131,19 +127,10 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
         assertThat(bytesValuesIter.hasNext(), equalTo(false));
 
-        bytesValuesIter = bytesValues.getIterSafe(0);
-        assertThat(bytesValuesIter.hasNext(), equalTo(true));
-        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
-        assertThat(bytesValuesIter.hasNext(), equalTo(false));
-
         bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
         bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
         bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
 
-        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
-        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
-        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
-
         HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
@@ -154,28 +141,15 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
         assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
 
-        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
-        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
-
         HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
         assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
 
-        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
-        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
-
         hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
         hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
         hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
 
-        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
-        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
-        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
-
         StringValues stringValues = fieldData.getStringValues();
 
         assertThat(stringValues.hasValue(0), equalTo(true));
@@ -277,10 +251,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(bytesValues.getValue(1), nullValue());
         assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
 
-        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
-        assertThat(bytesValues.getValueSafe(1), nullValue());
-        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
-
         BytesRef bytesRef = new BytesRef();
         assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
         assertThat(bytesRef, equalTo(new BytesRef(two())));
@@ -309,22 +279,10 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         bytesValuesIter = bytesValues.getIter(1);
         assertThat(bytesValuesIter.hasNext(), equalTo(false));
 
-        bytesValuesIter = bytesValues.getIterSafe(0);
-        assertThat(bytesValuesIter.hasNext(), equalTo(true));
-        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
-        assertThat(bytesValuesIter.hasNext(), equalTo(false));
-
-        bytesValuesIter = bytesValues.getIterSafe(1);
-        assertThat(bytesValuesIter.hasNext(), equalTo(false));
-
         bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
         bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
         bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
 
-        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
-        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
-        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
-
         HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
@@ -335,10 +293,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(hashedBytesValues.getValue(1), nullValue());
         assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
 
-        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValues.getValueSafe(1), nullValue());
-        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
-
         HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
         assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
@@ -347,22 +301,10 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         hashedBytesValuesIter = hashedBytesValues.getIter(1);
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
 
-        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
-        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
-
-        hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
-
         hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
         hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
         hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
 
-        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
-        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
-        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
-
         StringValues stringValues = fieldData.getStringValues();
 
         assertThat(stringValues.hasValue(0), equalTo(true));
@@ -443,10 +385,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
         assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
 
-        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
-        assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
-        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
-
         BytesRef bytesRef = new BytesRef();
         assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
         assertThat(bytesRef, equalTo(new BytesRef(two())));
@@ -476,21 +414,10 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
         assertThat(bytesValuesIter.hasNext(), equalTo(false));
 
-        bytesValuesIter = bytesValues.getIterSafe(0);
-        assertThat(bytesValuesIter.hasNext(), equalTo(true));
-        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
-        assertThat(bytesValuesIter.hasNext(), equalTo(true));
-        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
-        assertThat(bytesValuesIter.hasNext(), equalTo(false));
-
         bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
         bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
         bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
 
-        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
-        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
-        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
-
         HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
@@ -501,10 +428,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
         assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
 
-        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
-        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
-
         HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
         assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
@@ -512,21 +435,10 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
 
-        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
-        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
-        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
-
         hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
         hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
         hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
 
-        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
-        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
-        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
-
         StringValues stringValues = fieldData.getStringValues();
 
         assertThat(stringValues.hasValue(0), equalTo(true));
@@ -611,10 +523,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(bytesValues.getValue(1), nullValue());
         assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
 
-        assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
-        assertThat(bytesValues.getValueSafe(1), nullValue());
-        assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
-
         BytesRef bytesRef = new BytesRef();
         assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
         assertThat(bytesRef, equalTo(new BytesRef(two())));
@@ -646,24 +554,10 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         bytesValuesIter = bytesValues.getIter(1);
         assertThat(bytesValuesIter.hasNext(), equalTo(false));
 
-        bytesValuesIter = bytesValues.getIterSafe(0);
-        assertThat(bytesValuesIter.hasNext(), equalTo(true));
-        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
-        assertThat(bytesValuesIter.hasNext(), equalTo(true));
-        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
-        assertThat(bytesValuesIter.hasNext(), equalTo(false));
-
-        bytesValuesIter = bytesValues.getIterSafe(1);
-        assertThat(bytesValuesIter.hasNext(), equalTo(false));
-
         bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
         bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
         bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
 
-        bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
-        bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
-        bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
-
         HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
@@ -674,10 +568,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         assertThat(hashedBytesValues.getValue(1), nullValue());
         assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
 
-        assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValues.getValueSafe(1), nullValue());
-        assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
-
         HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
         assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
@@ -688,24 +578,10 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         hashedBytesValuesIter = hashedBytesValues.getIter(1);
         assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
 
-        hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
-        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
-        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
-
-        hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
-        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
-
         hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
         hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
         hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
 
-        hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
-        hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
-        hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
-
         StringValues stringValues = fieldData.getStringValues();
 
         assertThat(stringValues.hasValue(0), equalTo(true));

From 772ee9db54c26bd154a837c7ea02b5a7316c92f1 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 12:06:26 +0100
Subject: [PATCH 27/35] move terms to use new field data

---
 .../fielddata/ordinals/EmptyOrdinals.java     |   2 +-
 .../facet/terms/InternalTermsFacet.java       |  10 -
 .../facet/terms/TermsFacetProcessor.java      |  94 ++----
 .../terms/bytes/InternalByteTermsFacet.java   | 312 -----------------
 .../terms/bytes/TermsByteFacetCollector.java  | 255 --------------
 .../TermsByteOrdinalsFacetCollector.java      | 268 ---------------
 .../doubles/TermsDoubleFacetCollector.java    |  78 ++---
 .../TermsDoubleOrdinalsFacetCollector.java    | 267 ---------------
 .../terms/floats/InternalFloatTermsFacet.java | 311 -----------------
 .../floats/TermsFloatFacetCollector.java      | 254 --------------
 .../TermsFloatOrdinalsFacetCollector.java     | 267 ---------------
 .../terms/ints/InternalIntTermsFacet.java     | 308 -----------------
 .../terms/ints/TermsIntFacetCollector.java    | 254 --------------
 .../ints/TermsIntOrdinalsFacetCollector.java  | 267 ---------------
 .../facet/terms/ip/InternalIpTermsFacet.java  | 313 ------------------
 .../facet/terms/ip/TermsIpFacetCollector.java | 237 -------------
 .../ip/TermsIpOrdinalsFacetCollector.java     | 266 ---------------
 .../terms/longs/TermsLongFacetCollector.java  |  91 ++---
 .../TermsLongOrdinalsFacetCollector.java      | 267 ---------------
 .../terms/shorts/InternalShortTermsFacet.java | 308 -----------------
 .../shorts/TermsShortFacetCollector.java      | 255 --------------
 .../TermsShortOrdinalsFacetCollector.java     | 267 ---------------
 .../FieldsTermsStringFacetCollector.java      | 147 ++++----
 .../strings/TermsStringFacetCollector.java    | 122 +++----
 .../TermsStringOrdinalsFacetCollector.java    |  65 ++--
 .../search/facet/SimpleFacetsTests.java       |  35 +-
 26 files changed, 224 insertions(+), 5096 deletions(-)
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteOrdinalsFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleOrdinalsFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatOrdinalsFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntOrdinalsFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/ip/InternalIpTermsFacet.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpOrdinalsFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongOrdinalsFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java
 delete mode 100644 src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortOrdinalsFacetCollector.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java
index b9d6b9c7e14..5049fbd1c52 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java
@@ -53,7 +53,7 @@ public class EmptyOrdinals implements Ordinals {
 
     @Override
     public int getNumOrds() {
-        return numDocs + 1;
+        return 1;
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/InternalTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/InternalTermsFacet.java
index bcc6a673d63..d9bdea3704f 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/InternalTermsFacet.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/InternalTermsFacet.java
@@ -21,13 +21,8 @@ package org.elasticsearch.search.facet.terms;
 
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.InternalFacet;
-import org.elasticsearch.search.facet.terms.bytes.InternalByteTermsFacet;
 import org.elasticsearch.search.facet.terms.doubles.InternalDoubleTermsFacet;
-import org.elasticsearch.search.facet.terms.floats.InternalFloatTermsFacet;
-import org.elasticsearch.search.facet.terms.ints.InternalIntTermsFacet;
-import org.elasticsearch.search.facet.terms.ip.InternalIpTermsFacet;
 import org.elasticsearch.search.facet.terms.longs.InternalLongTermsFacet;
-import org.elasticsearch.search.facet.terms.shorts.InternalShortTermsFacet;
 import org.elasticsearch.search.facet.terms.strings.InternalStringTermsFacet;
 
 import java.util.List;
@@ -41,11 +36,6 @@ public abstract class InternalTermsFacet implements TermsFacet, InternalFacet {
         InternalStringTermsFacet.registerStream();
         InternalLongTermsFacet.registerStream();
         InternalDoubleTermsFacet.registerStream();
-        InternalIntTermsFacet.registerStream();
-        InternalFloatTermsFacet.registerStream();
-        InternalShortTermsFacet.registerStream();
-        InternalByteTermsFacet.registerStream();
-        InternalIpTermsFacet.registerStream();
     }
 
     public abstract Facet reduce(String name, List<Facet> facets);
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/TermsFacetProcessor.java b/src/main/java/org/elasticsearch/search/facet/terms/TermsFacetProcessor.java
index c002baec39d..c51cb272e0a 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/TermsFacetProcessor.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/TermsFacetProcessor.java
@@ -27,27 +27,18 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.ip.IpFieldMapper;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.FacetCollector;
+import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.FacetProcessor;
-import org.elasticsearch.search.facet.terms.bytes.TermsByteFacetCollector;
-import org.elasticsearch.search.facet.terms.bytes.TermsByteOrdinalsFacetCollector;
 import org.elasticsearch.search.facet.terms.doubles.TermsDoubleFacetCollector;
-import org.elasticsearch.search.facet.terms.doubles.TermsDoubleOrdinalsFacetCollector;
-import org.elasticsearch.search.facet.terms.floats.TermsFloatFacetCollector;
-import org.elasticsearch.search.facet.terms.floats.TermsFloatOrdinalsFacetCollector;
 import org.elasticsearch.search.facet.terms.index.IndexNameFacetCollector;
-import org.elasticsearch.search.facet.terms.ints.TermsIntFacetCollector;
-import org.elasticsearch.search.facet.terms.ints.TermsIntOrdinalsFacetCollector;
-import org.elasticsearch.search.facet.terms.ip.TermsIpFacetCollector;
-import org.elasticsearch.search.facet.terms.ip.TermsIpOrdinalsFacetCollector;
 import org.elasticsearch.search.facet.terms.longs.TermsLongFacetCollector;
-import org.elasticsearch.search.facet.terms.longs.TermsLongOrdinalsFacetCollector;
-import org.elasticsearch.search.facet.terms.shorts.TermsShortFacetCollector;
-import org.elasticsearch.search.facet.terms.shorts.TermsShortOrdinalsFacetCollector;
 import org.elasticsearch.search.facet.terms.strings.FieldsTermsStringFacetCollector;
 import org.elasticsearch.search.facet.terms.strings.ScriptTermsStringFieldFacetCollector;
 import org.elasticsearch.search.facet.terms.strings.TermsStringFacetCollector;
@@ -147,64 +138,41 @@ public class TermsFacetProcessor extends AbstractComponent implements FacetProce
         if (regex != null) {
             pattern = Regex.compile(regex, regexFlags);
         }
+
+        SearchScript searchScript = null;
+        if (script != null) {
+            searchScript = context.scriptService().search(context.lookup(), scriptLang, script, params);
+        }
+
         if (fieldsNames != null) {
-            return new FieldsTermsStringFacetCollector(facetName, fieldsNames, size, comparatorType, allTerms, context, excluded, pattern, scriptLang, script, params);
+            return new FieldsTermsStringFacetCollector(facetName, fieldsNames, size, comparatorType, allTerms, context, excluded, pattern, searchScript);
         }
         if (field == null && fieldsNames == null && script != null) {
             return new ScriptTermsStringFieldFacetCollector(facetName, size, comparatorType, context, excluded, pattern, scriptLang, script, params);
         }
 
         FieldMapper fieldMapper = context.smartNameFieldMapper(field);
-        if (fieldMapper != null) {
-            if (fieldMapper instanceof IpFieldMapper) {
-                if (script != null || "map".equals(executionHint)) {
-                    return new TermsIpFacetCollector(facetName, field, size, comparatorType, allTerms, context, scriptLang, script, params);
-                } else {
-                    return new TermsIpOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, null);
-                }
-            } else if (fieldMapper.fieldDataType() == FieldDataType.DefaultTypes.LONG) {
-                if (script != null || "map".equals(executionHint)) {
-                    return new TermsLongFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, scriptLang, script, params);
-                } else {
-                    return new TermsLongOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded);
-                }
-            } else if (fieldMapper.fieldDataType() == FieldDataType.DefaultTypes.DOUBLE) {
-                if (script != null) {
-                    return new TermsDoubleFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, scriptLang, script, params);
-                } else {
-                    return new TermsDoubleOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded);
-                }
-            } else if (fieldMapper.fieldDataType() == FieldDataType.DefaultTypes.INT) {
-                if (script != null || "map".equals(executionHint)) {
-                    return new TermsIntFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, scriptLang, script, params);
-                } else {
-                    return new TermsIntOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded);
-                }
-            } else if (fieldMapper.fieldDataType() == FieldDataType.DefaultTypes.FLOAT) {
-                if (script != null || "map".equals(executionHint)) {
-                    return new TermsFloatFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, scriptLang, script, params);
-                } else {
-                    return new TermsFloatOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded);
-                }
-            } else if (fieldMapper.fieldDataType() == FieldDataType.DefaultTypes.SHORT) {
-                if (script != null || "map".equals(executionHint)) {
-                    return new TermsShortFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, scriptLang, script, params);
-                } else {
-                    return new TermsShortOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded);
-                }
-            } else if (fieldMapper.fieldDataType() == FieldDataType.DefaultTypes.BYTE) {
-                if (script != null || "map".equals(executionHint)) {
-                    return new TermsByteFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, scriptLang, script, params);
-                } else {
-                    return new TermsByteOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded);
-                }
-            } else if (fieldMapper.fieldDataType() == FieldDataType.DefaultTypes.STRING) {
-                if (script == null && !"map".equals(executionHint)) {
-                    return new TermsStringOrdinalsFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, pattern);
-                }
+        if (fieldMapper == null) {
+            throw new FacetPhaseExecutionException(facetName, "failed to find mapping for [" + field + "]");
+        }
+
+        IndexFieldData indexFieldData = context.fieldData().getForField(fieldMapper);
+        if (indexFieldData instanceof IndexNumericFieldData) {
+            IndexNumericFieldData indexNumericFieldData = (IndexNumericFieldData) indexFieldData;
+            if (indexNumericFieldData.getNumericType().isFloatingPoint()) {
+                return new TermsDoubleFacetCollector(facetName, indexNumericFieldData, size, comparatorType, allTerms, context, excluded, searchScript);
+            } else {
+                return new TermsLongFacetCollector(facetName, indexNumericFieldData, size, comparatorType, allTerms, context, excluded, searchScript);
+            }
+        } else {
+            if (script != null || "map".equals(executionHint)) {
+                return new TermsStringFacetCollector(facetName, indexFieldData, size, comparatorType, allTerms, context, excluded, pattern, searchScript);
+            } else if (indexFieldData instanceof IndexOrdinalFieldData) {
+                return new TermsStringOrdinalsFacetCollector(facetName, (IndexOrdinalFieldData) indexFieldData, size, comparatorType, allTerms, context, excluded, pattern);
+            } else {
+                return new TermsStringFacetCollector(facetName, indexFieldData, size, comparatorType, allTerms, context, excluded, pattern, searchScript);
             }
         }
-        return new TermsStringFacetCollector(facetName, field, size, comparatorType, allTerms, context, excluded, pattern, scriptLang, script, params);
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java
deleted file mode 100644
index df970366235..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.bytes;
-
-import com.google.common.collect.ImmutableList;
-import gnu.trove.iterator.TByteIntIterator;
-import gnu.trove.map.hash.TByteIntHashMap;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.text.StringText;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.InternalTermsFacet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class InternalByteTermsFacet extends InternalTermsFacet {
-
-    private static final String STREAM_TYPE = "bTerms";
-
-    public static void registerStream() {
-        Streams.registerStream(STREAM, STREAM_TYPE);
-    }
-
-    static Stream STREAM = new Stream() {
-        @Override
-        public Facet readFacet(String type, StreamInput in) throws IOException {
-            return readTermsFacet(in);
-        }
-    };
-
-    @Override
-    public String streamType() {
-        return STREAM_TYPE;
-    }
-
-    public static class ByteEntry implements Entry {
-
-        byte term;
-        int count;
-
-        public ByteEntry(byte term, int count) {
-            this.term = term;
-            this.count = count;
-        }
-
-        public Text term() {
-            return new StringText(Short.toString(term));
-        }
-
-        public Text getTerm() {
-            return term();
-        }
-
-        @Override
-        public Number termAsNumber() {
-            return term;
-        }
-
-        @Override
-        public Number getTermAsNumber() {
-            return termAsNumber();
-        }
-
-        public int count() {
-            return count;
-        }
-
-        public int getCount() {
-            return count();
-        }
-
-        @Override
-        public int compareTo(Entry o) {
-            byte anotherVal = ((ByteEntry) o).term;
-            int i = term - anotherVal;
-            if (i == 0) {
-                i = count - o.count();
-                if (i == 0) {
-                    i = System.identityHashCode(this) - System.identityHashCode(o);
-                }
-            }
-            return i;
-        }
-    }
-
-    private String name;
-
-    int requiredSize;
-
-    long missing;
-    long total;
-
-    Collection<ByteEntry> entries = ImmutableList.of();
-
-    ComparatorType comparatorType;
-
-    InternalByteTermsFacet() {
-    }
-
-    public InternalByteTermsFacet(String name, ComparatorType comparatorType, int requiredSize, Collection<ByteEntry> entries, long missing, long total) {
-        this.name = name;
-        this.comparatorType = comparatorType;
-        this.requiredSize = requiredSize;
-        this.entries = entries;
-        this.missing = missing;
-        this.total = total;
-    }
-
-    @Override
-    public String name() {
-        return this.name;
-    }
-
-    @Override
-    public String getName() {
-        return this.name;
-    }
-
-    @Override
-    public String type() {
-        return TYPE;
-    }
-
-    @Override
-    public String getType() {
-        return type();
-    }
-
-    @Override
-    public long missingCount() {
-        return this.missing;
-    }
-
-    @Override
-    public long getMissingCount() {
-        return missingCount();
-    }
-
-    @Override
-    public long totalCount() {
-        return this.total;
-    }
-
-    @Override
-    public long getTotalCount() {
-        return totalCount();
-    }
-
-    @Override
-    public long otherCount() {
-        long other = total;
-        for (Entry entry : entries) {
-            other -= entry.count();
-        }
-        return other;
-    }
-
-    @Override
-    public long getOtherCount() {
-        return otherCount();
-    }
-
-
-    @Override
-    public List<ByteEntry> entries() {
-        if (!(entries instanceof List)) {
-            entries = ImmutableList.copyOf(entries);
-        }
-        return (List<ByteEntry>) entries;
-    }
-
-    @Override
-    public List<ByteEntry> getEntries() {
-        return entries();
-    }
-
-    @SuppressWarnings({"unchecked"})
-    @Override
-    public Iterator<Entry> iterator() {
-        return (Iterator) entries.iterator();
-    }
-
-
-    @Override
-    public Facet reduce(String name, List<Facet> facets) {
-        if (facets.size() == 1) {
-            return facets.get(0);
-        }
-        InternalByteTermsFacet first = (InternalByteTermsFacet) facets.get(0);
-        TByteIntHashMap aggregated = CacheRecycler.popByteIntMap();
-
-        long missing = 0;
-        long total = 0;
-        for (Facet facet : facets) {
-            InternalByteTermsFacet mFacet = (InternalByteTermsFacet) facet;
-            missing += mFacet.missingCount();
-            total += mFacet.totalCount();
-            for (ByteEntry entry : mFacet.entries) {
-                aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
-            }
-        }
-
-        BoundedTreeSet<ByteEntry> ordered = new BoundedTreeSet<ByteEntry>(first.comparatorType.comparator(), first.requiredSize);
-        for (TByteIntIterator it = aggregated.iterator(); it.hasNext(); ) {
-            it.advance();
-            ordered.add(new ByteEntry(it.key(), it.value()));
-        }
-        first.entries = ordered;
-        first.missing = missing;
-        first.total = total;
-
-        CacheRecycler.pushByteIntMap(aggregated);
-
-        return first;
-    }
-
-    static final class Fields {
-        static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
-        static final XContentBuilderString MISSING = new XContentBuilderString("missing");
-        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-        static final XContentBuilderString OTHER = new XContentBuilderString("other");
-        static final XContentBuilderString TERMS = new XContentBuilderString("terms");
-        static final XContentBuilderString TERM = new XContentBuilderString("term");
-        static final XContentBuilderString COUNT = new XContentBuilderString("count");
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(name);
-        builder.field(Fields._TYPE, TermsFacet.TYPE);
-        builder.field(Fields.MISSING, missing);
-        builder.field(Fields.TOTAL, total);
-        builder.field(Fields.OTHER, otherCount());
-        builder.startArray(Fields.TERMS);
-        for (ByteEntry entry : entries) {
-            builder.startObject();
-            builder.field(Fields.TERM, entry.term);
-            builder.field(Fields.COUNT, entry.count());
-            builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-    public static InternalByteTermsFacet readTermsFacet(StreamInput in) throws IOException {
-        InternalByteTermsFacet facet = new InternalByteTermsFacet();
-        facet.readFrom(in);
-        return facet;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        comparatorType = ComparatorType.fromId(in.readByte());
-        requiredSize = in.readVInt();
-        missing = in.readVLong();
-        total = in.readVLong();
-
-        int size = in.readVInt();
-        entries = new ArrayList<ByteEntry>(size);
-        for (int i = 0; i < size; i++) {
-            entries.add(new ByteEntry(in.readByte(), in.readVInt()));
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeByte(comparatorType.id());
-
-        out.writeVInt(requiredSize);
-        out.writeVLong(missing);
-        out.writeVLong(total);
-
-        out.writeVInt(entries.size());
-        for (ByteEntry entry : entries) {
-            out.writeByte(entry.term);
-            out.writeVInt(entry.count());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java
deleted file mode 100644
index 92bc191ee39..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteFacetCollector.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.bytes;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.iterator.TByteIntIterator;
-import gnu.trove.map.hash.TByteIntHashMap;
-import gnu.trove.set.hash.TByteHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.bytes.ByteFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Set;
-
-/**
- *
- */
-public class TermsByteFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final FieldDataType fieldDataType;
-
-    private ByteFieldData fieldData;
-
-    private final StaticAggregatorValueProc aggregator;
-
-    private final SearchScript script;
-
-    public TermsByteFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                   ImmutableSet<BytesRef> excluded, String scriptLang, String script, Map<String, Object> params) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms short facet collector on it");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.BYTE) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of byte type, can't run terms short facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
-
-        if (this.script == null && excluded.isEmpty()) {
-            aggregator = new StaticAggregatorValueProc(CacheRecycler.popByteIntMap());
-        } else {
-            aggregator = new AggregatorValueProc(CacheRecycler.popByteIntMap(), excluded, this.script);
-        }
-
-        if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    ByteFieldData fieldData = (ByteFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
-        }
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) throws IOException {
-        if (script != null) {
-            script.setScorer(scorer);
-        }
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (ByteFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        if (script != null) {
-            script.setNextReader(context);
-        }
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
-    }
-
-    @Override
-    public Facet facet() {
-        TByteIntHashMap facets = aggregator.facets();
-        if (facets.isEmpty()) {
-            CacheRecycler.pushByteIntMap(facets);
-            return new InternalByteTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalByteTermsFacet.ByteEntry>of(), aggregator.missing(), aggregator.total());
-        } else {
-            if (size < EntryPriorityQueue.LIMIT) {
-                EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-                for (TByteIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.insertWithOverflow(new InternalByteTermsFacet.ByteEntry(it.key(), it.value()));
-                }
-                InternalByteTermsFacet.ByteEntry[] list = new InternalByteTermsFacet.ByteEntry[ordered.size()];
-                for (int i = ordered.size() - 1; i >= 0; i--) {
-                    list[i] = (InternalByteTermsFacet.ByteEntry) ordered.pop();
-                }
-                CacheRecycler.pushByteIntMap(facets);
-                return new InternalByteTermsFacet(facetName, comparatorType, size, Arrays.asList(list), aggregator.missing(), aggregator.total());
-            } else {
-                BoundedTreeSet<InternalByteTermsFacet.ByteEntry> ordered = new BoundedTreeSet<InternalByteTermsFacet.ByteEntry>(comparatorType.comparator(), size);
-                for (TByteIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.add(new InternalByteTermsFacet.ByteEntry(it.key(), it.value()));
-                }
-                CacheRecycler.pushByteIntMap(facets);
-                return new InternalByteTermsFacet(facetName, comparatorType, size, ordered, aggregator.missing(), aggregator.total());
-            }
-        }
-    }
-
-    public static class AggregatorValueProc extends StaticAggregatorValueProc {
-
-        private final SearchScript script;
-
-        private final TByteHashSet excluded;
-
-        public AggregatorValueProc(TByteIntHashMap facets, Set<BytesRef> excluded, SearchScript script) {
-            super(facets);
-            if (excluded == null || excluded.isEmpty()) {
-                this.excluded = null;
-            } else {
-                this.excluded = new TByteHashSet(excluded.size());
-                for (BytesRef s : excluded) {
-                    this.excluded.add(Byte.parseByte(s.utf8ToString()));
-                }
-            }
-            this.script = script;
-        }
-
-        @Override
-        public void onValue(int docId, byte value) {
-            if (excluded != null && excluded.contains(value)) {
-                return;
-            }
-            if (script != null) {
-                script.setNextDocId(docId);
-                script.setNextVar("term", value);
-                Object scriptValue = script.run();
-                if (scriptValue == null) {
-                    return;
-                }
-                if (scriptValue instanceof Boolean) {
-                    if (!((Boolean) scriptValue)) {
-                        return;
-                    }
-                } else {
-                    value = ((Number) scriptValue).byteValue();
-                }
-            }
-            super.onValue(docId, value);
-        }
-    }
-
-    public static class StaticAggregatorValueProc implements ByteFieldData.ValueInDocProc, ByteFieldData.ValueProc {
-
-        private final TByteIntHashMap facets;
-
-        private int missing;
-        private int total;
-
-        public StaticAggregatorValueProc(TByteIntHashMap facets) {
-            this.facets = facets;
-        }
-
-        @Override
-        public void onValue(byte value) {
-            facets.putIfAbsent(value, 0);
-        }
-
-        @Override
-        public void onValue(int docId, byte value) {
-            facets.adjustOrPutValue(value, 1, 1);
-            total++;
-        }
-
-        @Override
-        public void onMissing(int docID) {
-            missing++;
-        }
-
-        public final TByteIntHashMap facets() {
-            return facets;
-        }
-
-        public final int missing() {
-            return this.missing;
-        }
-
-        public final int total() {
-            return this.total;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteOrdinalsFacetCollector.java
deleted file mode 100644
index f4571d8f3c4..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/bytes/TermsByteOrdinalsFacetCollector.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.bytes;
-
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.set.hash.TByteHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.bytes.ByteFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- *
- */
-public class TermsByteOrdinalsFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final int minCount;
-
-    private final FieldDataType fieldDataType;
-
-    private ByteFieldData fieldData;
-
-    private final List<ReaderAggregator> aggregators;
-
-    private ReaderAggregator current;
-
-    long missing;
-    long total;
-
-    private final TByteHashSet excluded;
-
-    public TermsByteOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                           ImmutableSet<BytesRef> excluded) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms byte facet collector on it");
-        }
-
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.BYTE) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of byte type, can't run terms byte facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (excluded == null || excluded.isEmpty()) {
-            this.excluded = null;
-        } else {
-            this.excluded = new TByteHashSet(excluded.size());
-            for (BytesRef s : excluded) {
-                this.excluded.add(Byte.parseByte(s.utf8ToString()));
-            }
-        }
-
-        // minCount is offset by -1
-        if (allTerms) {
-            minCount = -1;
-        } else {
-            minCount = 0;
-        }
-
-        this.aggregators = new ArrayList<ReaderAggregator>(context.searcher().getIndexReader().leaves().size());
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-        fieldData = (ByteFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
-    }
-
-    @Override
-    public Facet facet() {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            // if we have values for this one, add it
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-
-        AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
-
-        for (ReaderAggregator aggregator : aggregators) {
-            if (aggregator.nextPosition()) {
-                queue.add(aggregator);
-            }
-        }
-
-        // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
-        if (size < EntryPriorityQueue.LIMIT) {
-            // optimize to use priority size
-            EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-
-            while (queue.size() > 0) {
-                ReaderAggregator agg = queue.top();
-                byte value = agg.current;
-                int count = 0;
-                do {
-                    count += agg.counts[agg.position];
-                    if (agg.nextPosition()) {
-                        agg = queue.updateTop();
-                    } else {
-                        // we are done with this reader
-                        queue.pop();
-                        agg = queue.top();
-                    }
-                } while (agg != null && value == agg.current);
-
-                if (count > minCount) {
-                    if (excluded == null || !excluded.contains(value)) {
-                        InternalByteTermsFacet.ByteEntry entry = new InternalByteTermsFacet.ByteEntry(value, count);
-                        ordered.insertWithOverflow(entry);
-                    }
-                }
-            }
-            InternalByteTermsFacet.ByteEntry[] list = new InternalByteTermsFacet.ByteEntry[ordered.size()];
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = (InternalByteTermsFacet.ByteEntry) ordered.pop();
-            }
-
-            for (ReaderAggregator aggregator : aggregators) {
-                CacheRecycler.pushIntArray(aggregator.counts);
-            }
-
-            return new InternalByteTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
-        }
-
-        BoundedTreeSet<InternalByteTermsFacet.ByteEntry> ordered = new BoundedTreeSet<InternalByteTermsFacet.ByteEntry>(comparatorType.comparator(), size);
-
-        while (queue.size() > 0) {
-            ReaderAggregator agg = queue.top();
-            byte value = agg.current;
-            int count = 0;
-            do {
-                count += agg.counts[agg.position];
-                if (agg.nextPosition()) {
-                    agg = queue.updateTop();
-                } else {
-                    // we are done with this reader
-                    queue.pop();
-                    agg = queue.top();
-                }
-            } while (agg != null && value == agg.current);
-
-            if (count > minCount) {
-                if (excluded == null || !excluded.contains(value)) {
-                    InternalByteTermsFacet.ByteEntry entry = new InternalByteTermsFacet.ByteEntry(value, count);
-                    ordered.add(entry);
-                }
-            }
-        }
-
-        for (ReaderAggregator aggregator : aggregators) {
-            CacheRecycler.pushIntArray(aggregator.counts);
-        }
-
-        return new InternalByteTermsFacet(facetName, comparatorType, size, ordered, missing, total);
-    }
-
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
-
-        final byte[] values;
-        final int[] counts;
-
-        int position = 0;
-        byte current;
-        int total;
-
-        public ReaderAggregator(ByteFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
-        }
-
-        @Override
-        public void onOrdinal(int docId, int ordinal) {
-            counts[ordinal]++;
-            total++;
-        }
-
-        public boolean nextPosition() {
-            if (++position >= values.length) {
-                return false;
-            }
-            current = values[position];
-            return true;
-        }
-    }
-
-    public static class AggregatorPriorityQueue extends PriorityQueue<ReaderAggregator> {
-
-        public AggregatorPriorityQueue(int size) {
-            super(size);
-        }
-
-        @Override
-        protected boolean lessThan(ReaderAggregator a, ReaderAggregator b) {
-            return a.current < b.current;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java
index cb8ec6efac6..dfc1663a87b 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleFacetCollector.java
@@ -27,24 +27,19 @@ import gnu.trove.set.hash.TDoubleHashSet;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.DoubleValues;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.terms.TermsFacet;
 import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -52,9 +47,7 @@ import java.util.Set;
  */
 public class TermsDoubleFacetCollector extends AbstractFacetCollector {
 
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final TermsFacet.ComparatorType comparatorType;
 
@@ -62,43 +55,20 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector {
 
     private final int numberOfShards;
 
-    private final FieldDataType fieldDataType;
-
-    private DoubleFieldData fieldData;
+    private DoubleValues values;
 
     private final StaticAggregatorValueProc aggregator;
 
     private final SearchScript script;
 
-    public TermsDoubleFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                     ImmutableSet<BytesRef> excluded, String scriptLang, String script, Map<String, Object> params) {
+    public TermsDoubleFacetCollector(String facetName, IndexNumericFieldData indexFieldData, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
+                                     ImmutableSet<BytesRef> excluded, SearchScript script) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
+        this.indexFieldData = indexFieldData;
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms double facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.DOUBLE) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of double type, can't run terms double facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
+        this.script = script;
 
         if (this.script == null && excluded.isEmpty()) {
             aggregator = new StaticAggregatorValueProc(CacheRecycler.popDoubleIntMap());
@@ -106,16 +76,17 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector {
             aggregator = new AggregatorValueProc(CacheRecycler.popDoubleIntMap(), excluded, this.script);
         }
 
-        if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    DoubleFieldData fieldData = (DoubleFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
-        }
+        // TODO: we need to support this with the new field data....
+//        if (allTerms) {
+//            try {
+//                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
+//                    DoubleFieldData fieldData = (DoubleFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
+//                    fieldData.forEachValue(aggregator);
+//                }
+//            } catch (Exception e) {
+//                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
+//            }
+//        }
     }
 
     @Override
@@ -127,7 +98,7 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (DoubleFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getDoubleValues();
         if (script != null) {
             script.setNextReader(context);
         }
@@ -135,7 +106,7 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
+        values.forEachValueInDoc(doc, aggregator);
     }
 
     @Override
@@ -212,7 +183,7 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector {
         }
     }
 
-    public static class StaticAggregatorValueProc implements DoubleFieldData.ValueInDocProc, DoubleFieldData.ValueProc {
+    public static class StaticAggregatorValueProc implements DoubleValues.ValueInDocProc {
 
         private final TDoubleIntHashMap facets;
 
@@ -223,11 +194,6 @@ public class TermsDoubleFacetCollector extends AbstractFacetCollector {
             this.facets = facets;
         }
 
-        @Override
-        public void onValue(double value) {
-            facets.putIfAbsent(value, 0);
-        }
-
         @Override
         public void onValue(int docId, double value) {
             facets.adjustOrPutValue(value, 1, 1);
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleOrdinalsFacetCollector.java
deleted file mode 100644
index 5e2331bca7b..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/doubles/TermsDoubleOrdinalsFacetCollector.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.doubles;
-
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.set.hash.TDoubleHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- *
- */
-public class TermsDoubleOrdinalsFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final int minCount;
-
-    private final FieldDataType fieldDataType;
-
-    private DoubleFieldData fieldData;
-
-    private final List<ReaderAggregator> aggregators;
-
-    private ReaderAggregator current;
-
-    long missing;
-    long total;
-
-    private final TDoubleHashSet excluded;
-
-    public TermsDoubleOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                             ImmutableSet<BytesRef> excluded) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms double facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.DOUBLE) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of double type, can't run terms double facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (excluded == null || excluded.isEmpty()) {
-            this.excluded = null;
-        } else {
-            this.excluded = new TDoubleHashSet(excluded.size());
-            for (BytesRef s : excluded) {
-                this.excluded.add(Double.parseDouble(s.utf8ToString()));
-            }
-        }
-
-        // minCount is offset by -1
-        if (allTerms) {
-            minCount = -1;
-        } else {
-            minCount = 0;
-        }
-
-        this.aggregators = new ArrayList<ReaderAggregator>(context.searcher().getIndexReader().leaves().size());
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-        fieldData = (DoubleFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
-    }
-
-    @Override
-    public Facet facet() {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            // if we have values for this one, add it
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-
-        AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
-
-        for (ReaderAggregator aggregator : aggregators) {
-            if (aggregator.nextPosition()) {
-                queue.add(aggregator);
-            }
-        }
-
-        // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
-        if (size < EntryPriorityQueue.LIMIT) {
-            // optimize to use priority size
-            EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-
-            while (queue.size() > 0) {
-                ReaderAggregator agg = queue.top();
-                double value = agg.current;
-                int count = 0;
-                do {
-                    count += agg.counts[agg.position];
-                    if (agg.nextPosition()) {
-                        agg = queue.updateTop();
-                    } else {
-                        // we are done with this reader
-                        queue.pop();
-                        agg = queue.top();
-                    }
-                } while (agg != null && value == agg.current);
-
-                if (count > minCount) {
-                    if (excluded == null || !excluded.contains(value)) {
-                        InternalDoubleTermsFacet.DoubleEntry entry = new InternalDoubleTermsFacet.DoubleEntry(value, count);
-                        ordered.insertWithOverflow(entry);
-                    }
-                }
-            }
-            InternalDoubleTermsFacet.DoubleEntry[] list = new InternalDoubleTermsFacet.DoubleEntry[ordered.size()];
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = (InternalDoubleTermsFacet.DoubleEntry) ordered.pop();
-            }
-
-            for (ReaderAggregator aggregator : aggregators) {
-                CacheRecycler.pushIntArray(aggregator.counts);
-            }
-
-            return new InternalDoubleTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
-        }
-
-        BoundedTreeSet<InternalDoubleTermsFacet.DoubleEntry> ordered = new BoundedTreeSet<InternalDoubleTermsFacet.DoubleEntry>(comparatorType.comparator(), size);
-
-        while (queue.size() > 0) {
-            ReaderAggregator agg = queue.top();
-            double value = agg.current;
-            int count = 0;
-            do {
-                count += agg.counts[agg.position];
-                if (agg.nextPosition()) {
-                    agg = queue.updateTop();
-                } else {
-                    // we are done with this reader
-                    queue.pop();
-                    agg = queue.top();
-                }
-            } while (agg != null && value == agg.current);
-
-            if (count > minCount) {
-                if (excluded == null || !excluded.contains(value)) {
-                    InternalDoubleTermsFacet.DoubleEntry entry = new InternalDoubleTermsFacet.DoubleEntry(value, count);
-                    ordered.add(entry);
-                }
-            }
-        }
-
-        for (ReaderAggregator aggregator : aggregators) {
-            CacheRecycler.pushIntArray(aggregator.counts);
-        }
-
-        return new InternalDoubleTermsFacet(facetName, comparatorType, size, ordered, missing, total);
-    }
-
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
-
-        final double[] values;
-        final int[] counts;
-
-        int position = 0;
-        double current;
-        int total;
-
-        public ReaderAggregator(DoubleFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
-        }
-
-        @Override
-        public void onOrdinal(int docId, int ordinal) {
-            counts[ordinal]++;
-            total++;
-        }
-
-        public boolean nextPosition() {
-            if (++position >= values.length) {
-                return false;
-            }
-            current = values[position];
-            return true;
-        }
-    }
-
-    public static class AggregatorPriorityQueue extends PriorityQueue<ReaderAggregator> {
-
-        public AggregatorPriorityQueue(int size) {
-            super(size);
-        }
-
-        @Override
-        protected boolean lessThan(ReaderAggregator a, ReaderAggregator b) {
-            return a.current < b.current;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java
deleted file mode 100644
index b16424e7f94..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.floats;
-
-import com.google.common.collect.ImmutableList;
-import gnu.trove.iterator.TFloatIntIterator;
-import gnu.trove.map.hash.TFloatIntHashMap;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.text.StringText;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.InternalTermsFacet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class InternalFloatTermsFacet extends InternalTermsFacet {
-
-    private static final String STREAM_TYPE = "fTerms";
-
-    public static void registerStream() {
-        Streams.registerStream(STREAM, STREAM_TYPE);
-    }
-
-    static Stream STREAM = new Stream() {
-        @Override
-        public Facet readFacet(String type, StreamInput in) throws IOException {
-            return readTermsFacet(in);
-        }
-    };
-
-    @Override
-    public String streamType() {
-        return STREAM_TYPE;
-    }
-
-    public static class FloatEntry implements Entry {
-
-        float term;
-        int count;
-
-        public FloatEntry(float term, int count) {
-            this.term = term;
-            this.count = count;
-        }
-
-        public Text term() {
-            return new StringText(Float.toString(term));
-        }
-
-        public Text getTerm() {
-            return term();
-        }
-
-        @Override
-        public Number termAsNumber() {
-            return term;
-        }
-
-        @Override
-        public Number getTermAsNumber() {
-            return termAsNumber();
-        }
-
-        public int count() {
-            return count;
-        }
-
-        public int getCount() {
-            return count();
-        }
-
-        @Override
-        public int compareTo(Entry o) {
-            float anotherVal = ((FloatEntry) o).term;
-            if (term < anotherVal) {
-                return -1;
-            }
-            if (term == anotherVal) {
-                int i = count - o.count();
-                if (i == 0) {
-                    i = System.identityHashCode(this) - System.identityHashCode(o);
-                }
-                return i;
-            }
-            return 1;
-        }
-    }
-
-    private String name;
-
-    int requiredSize;
-
-    long missing;
-    long total;
-
-    Collection<FloatEntry> entries = ImmutableList.of();
-
-    ComparatorType comparatorType;
-
-    InternalFloatTermsFacet() {
-    }
-
-    public InternalFloatTermsFacet(String name, ComparatorType comparatorType, int requiredSize, Collection<FloatEntry> entries, long missing, long total) {
-        this.name = name;
-        this.comparatorType = comparatorType;
-        this.requiredSize = requiredSize;
-        this.entries = entries;
-        this.missing = missing;
-        this.total = total;
-    }
-
-    @Override
-    public String name() {
-        return this.name;
-    }
-
-    @Override
-    public String getName() {
-        return this.name;
-    }
-
-    @Override
-    public String type() {
-        return TYPE;
-    }
-
-    @Override
-    public String getType() {
-        return type();
-    }
-
-    @Override
-    public List<FloatEntry> entries() {
-        if (!(entries instanceof List)) {
-            entries = ImmutableList.copyOf(entries);
-        }
-        return (List<FloatEntry>) entries;
-    }
-
-    @Override
-    public List<FloatEntry> getEntries() {
-        return entries();
-    }
-
-    @SuppressWarnings({"unchecked"})
-    @Override
-    public Iterator<Entry> iterator() {
-        return (Iterator) entries.iterator();
-    }
-
-    @Override
-    public long missingCount() {
-        return this.missing;
-    }
-
-    @Override
-    public long getMissingCount() {
-        return missingCount();
-    }
-
-    @Override
-    public long totalCount() {
-        return this.total;
-    }
-
-    @Override
-    public long getTotalCount() {
-        return totalCount();
-    }
-
-    @Override
-    public long otherCount() {
-        long other = total;
-        for (Entry entry : entries) {
-            other -= entry.count();
-        }
-        return other;
-    }
-
-    @Override
-    public long getOtherCount() {
-        return otherCount();
-    }
-
-    @Override
-    public Facet reduce(String name, List<Facet> facets) {
-        if (facets.size() == 1) {
-            return facets.get(0);
-        }
-        InternalFloatTermsFacet first = (InternalFloatTermsFacet) facets.get(0);
-        TFloatIntHashMap aggregated = CacheRecycler.popFloatIntMap();
-        long missing = 0;
-        long total = 0;
-        for (Facet facet : facets) {
-            InternalFloatTermsFacet mFacet = (InternalFloatTermsFacet) facet;
-            missing += mFacet.missingCount();
-            total += mFacet.totalCount();
-            for (FloatEntry entry : mFacet.entries) {
-                aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
-            }
-        }
-
-        BoundedTreeSet<FloatEntry> ordered = new BoundedTreeSet<FloatEntry>(first.comparatorType.comparator(), first.requiredSize);
-        for (TFloatIntIterator it = aggregated.iterator(); it.hasNext(); ) {
-            it.advance();
-            ordered.add(new FloatEntry(it.key(), it.value()));
-        }
-        first.entries = ordered;
-        first.missing = missing;
-        first.total = total;
-
-        CacheRecycler.pushFloatIntMap(aggregated);
-
-        return first;
-    }
-
-    static final class Fields {
-        static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
-        static final XContentBuilderString MISSING = new XContentBuilderString("missing");
-        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-        static final XContentBuilderString OTHER = new XContentBuilderString("other");
-        static final XContentBuilderString TERMS = new XContentBuilderString("terms");
-        static final XContentBuilderString TERM = new XContentBuilderString("term");
-        static final XContentBuilderString COUNT = new XContentBuilderString("count");
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(name);
-        builder.field(Fields._TYPE, TermsFacet.TYPE);
-        builder.field(Fields.MISSING, missing);
-        builder.field(Fields.TOTAL, total);
-        builder.field(Fields.OTHER, otherCount());
-        builder.startArray(Fields.TERMS);
-        for (FloatEntry entry : entries) {
-            builder.startObject();
-            builder.field(Fields.TERM, entry.term);
-            builder.field(Fields.COUNT, entry.count());
-            builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-    public static InternalFloatTermsFacet readTermsFacet(StreamInput in) throws IOException {
-        InternalFloatTermsFacet facet = new InternalFloatTermsFacet();
-        facet.readFrom(in);
-        return facet;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        comparatorType = ComparatorType.fromId(in.readByte());
-        requiredSize = in.readVInt();
-        missing = in.readVLong();
-        total = in.readVLong();
-
-        int size = in.readVInt();
-        entries = new ArrayList<FloatEntry>(size);
-        for (int i = 0; i < size; i++) {
-            entries.add(new FloatEntry(in.readFloat(), in.readVInt()));
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeByte(comparatorType.id());
-        out.writeVInt(requiredSize);
-        out.writeVLong(missing);
-        out.writeVLong(total);
-
-        out.writeVInt(entries.size());
-        for (FloatEntry entry : entries) {
-            out.writeFloat(entry.term);
-            out.writeVInt(entry.count());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java
deleted file mode 100644
index 5aef46adb61..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatFacetCollector.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.floats;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.iterator.TFloatIntIterator;
-import gnu.trove.map.hash.TFloatIntHashMap;
-import gnu.trove.set.hash.TFloatHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.floats.FloatFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Set;
-
-/**
- *
- */
-public class TermsFloatFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final FieldDataType fieldDataType;
-
-    private FloatFieldData fieldData;
-
-    private final StaticAggregatorValueProc aggregator;
-
-    private final SearchScript script;
-
-    public TermsFloatFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                    ImmutableSet<BytesRef> excluded, String scriptLang, String script, Map<String, Object> params) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms float facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.FLOAT) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't is not of float type, can't run terms float facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
-
-        if (this.script == null && excluded.isEmpty()) {
-            aggregator = new StaticAggregatorValueProc(CacheRecycler.popFloatIntMap());
-        } else {
-            aggregator = new AggregatorValueProc(CacheRecycler.popFloatIntMap(), excluded, this.script);
-        }
-
-        if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    FloatFieldData fieldData = (FloatFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
-        }
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) throws IOException {
-        if (script != null) {
-            script.setScorer(scorer);
-        }
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (FloatFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        if (script != null) {
-            script.setNextReader(context);
-        }
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
-    }
-
-    @Override
-    public Facet facet() {
-        TFloatIntHashMap facets = aggregator.facets();
-        if (facets.isEmpty()) {
-            CacheRecycler.pushFloatIntMap(facets);
-            return new InternalFloatTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalFloatTermsFacet.FloatEntry>of(), aggregator.missing(), aggregator.total());
-        } else {
-            if (size < EntryPriorityQueue.LIMIT) {
-                EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-                for (TFloatIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.insertWithOverflow(new InternalFloatTermsFacet.FloatEntry(it.key(), it.value()));
-                }
-                InternalFloatTermsFacet.FloatEntry[] list = new InternalFloatTermsFacet.FloatEntry[ordered.size()];
-                for (int i = ordered.size() - 1; i >= 0; i--) {
-                    list[i] = (InternalFloatTermsFacet.FloatEntry) ordered.pop();
-                }
-                CacheRecycler.pushFloatIntMap(facets);
-                return new InternalFloatTermsFacet(facetName, comparatorType, size, Arrays.asList(list), aggregator.missing(), aggregator.total());
-            } else {
-                BoundedTreeSet<InternalFloatTermsFacet.FloatEntry> ordered = new BoundedTreeSet<InternalFloatTermsFacet.FloatEntry>(comparatorType.comparator(), size);
-                for (TFloatIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.add(new InternalFloatTermsFacet.FloatEntry(it.key(), it.value()));
-                }
-                CacheRecycler.pushFloatIntMap(facets);
-                return new InternalFloatTermsFacet(facetName, comparatorType, size, ordered, aggregator.missing(), aggregator.total());
-            }
-        }
-    }
-
-    public static class AggregatorValueProc extends StaticAggregatorValueProc {
-
-        private final SearchScript script;
-
-        private final TFloatHashSet excluded;
-
-        public AggregatorValueProc(TFloatIntHashMap facets, Set<BytesRef> excluded, SearchScript script) {
-            super(facets);
-            if (excluded == null || excluded.isEmpty()) {
-                this.excluded = null;
-            } else {
-                this.excluded = new TFloatHashSet(excluded.size());
-                for (BytesRef s : excluded) {
-                    this.excluded.add(Float.parseFloat(s.utf8ToString()));
-                }
-            }
-            this.script = script;
-        }
-
-        @Override
-        public void onValue(int docId, float value) {
-            if (excluded != null && excluded.contains(value)) {
-                return;
-            }
-            if (script != null) {
-                script.setNextDocId(docId);
-                script.setNextVar("term", value);
-                Object scriptValue = script.run();
-                if (scriptValue == null) {
-                    return;
-                }
-                if (scriptValue instanceof Boolean) {
-                    if (!((Boolean) scriptValue)) {
-                        return;
-                    }
-                } else {
-                    value = ((Number) scriptValue).floatValue();
-                }
-            }
-            super.onValue(docId, value);
-        }
-    }
-
-    public static class StaticAggregatorValueProc implements FloatFieldData.ValueInDocProc, FloatFieldData.ValueProc {
-
-        private final TFloatIntHashMap facets;
-
-        private int missing;
-        private int total;
-
-        public StaticAggregatorValueProc(TFloatIntHashMap facets) {
-            this.facets = facets;
-        }
-
-        @Override
-        public void onValue(float value) {
-            facets.putIfAbsent(value, 0);
-        }
-
-        @Override
-        public void onValue(int docId, float value) {
-            facets.adjustOrPutValue(value, 1, 1);
-            total++;
-        }
-
-        @Override
-        public void onMissing(int docId) {
-            missing++;
-        }
-
-        public final TFloatIntHashMap facets() {
-            return facets;
-        }
-
-        public final int missing() {
-            return this.missing;
-        }
-
-        public final int total() {
-            return this.total;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatOrdinalsFacetCollector.java
deleted file mode 100644
index e5a717ec2a3..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/floats/TermsFloatOrdinalsFacetCollector.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.floats;
-
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.set.hash.TFloatHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.floats.FloatFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- *
- */
-public class TermsFloatOrdinalsFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final int minCount;
-
-    private final FieldDataType fieldDataType;
-
-    private FloatFieldData fieldData;
-
-    private final List<ReaderAggregator> aggregators;
-
-    private ReaderAggregator current;
-
-    long missing;
-    long total;
-
-    private final TFloatHashSet excluded;
-
-    public TermsFloatOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                            ImmutableSet<BytesRef> excluded) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms float facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.FLOAT) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of float type, can't run terms float facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (excluded == null || excluded.isEmpty()) {
-            this.excluded = null;
-        } else {
-            this.excluded = new TFloatHashSet(excluded.size());
-            for (BytesRef s : excluded) {
-                this.excluded.add(Float.parseFloat(s.utf8ToString()));
-            }
-        }
-
-        // minCount is offset by -1
-        if (allTerms) {
-            minCount = -1;
-        } else {
-            minCount = 0;
-        }
-
-        this.aggregators = new ArrayList<ReaderAggregator>(context.searcher().getIndexReader().leaves().size());
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-        fieldData = (FloatFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
-    }
-
-    @Override
-    public Facet facet() {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            // if we have values for this one, add it
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-
-        AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
-
-        for (ReaderAggregator aggregator : aggregators) {
-            if (aggregator.nextPosition()) {
-                queue.add(aggregator);
-            }
-        }
-
-        // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
-        if (size < EntryPriorityQueue.LIMIT) {
-            // optimize to use priority size
-            EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-
-            while (queue.size() > 0) {
-                ReaderAggregator agg = queue.top();
-                float value = agg.current;
-                int count = 0;
-                do {
-                    count += agg.counts[agg.position];
-                    if (agg.nextPosition()) {
-                        agg = queue.updateTop();
-                    } else {
-                        // we are done with this reader
-                        queue.pop();
-                        agg = queue.top();
-                    }
-                } while (agg != null && value == agg.current);
-
-                if (count > minCount) {
-                    if (excluded == null || !excluded.contains(value)) {
-                        InternalFloatTermsFacet.FloatEntry entry = new InternalFloatTermsFacet.FloatEntry(value, count);
-                        ordered.insertWithOverflow(entry);
-                    }
-                }
-            }
-            InternalFloatTermsFacet.FloatEntry[] list = new InternalFloatTermsFacet.FloatEntry[ordered.size()];
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = (InternalFloatTermsFacet.FloatEntry) ordered.pop();
-            }
-
-            for (ReaderAggregator aggregator : aggregators) {
-                CacheRecycler.pushIntArray(aggregator.counts);
-            }
-
-            return new InternalFloatTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
-        }
-
-        BoundedTreeSet<InternalFloatTermsFacet.FloatEntry> ordered = new BoundedTreeSet<InternalFloatTermsFacet.FloatEntry>(comparatorType.comparator(), size);
-
-        while (queue.size() > 0) {
-            ReaderAggregator agg = queue.top();
-            float value = agg.current;
-            int count = 0;
-            do {
-                count += agg.counts[agg.position];
-                if (agg.nextPosition()) {
-                    agg = queue.updateTop();
-                } else {
-                    // we are done with this reader
-                    queue.pop();
-                    agg = queue.top();
-                }
-            } while (agg != null && value == agg.current);
-
-            if (count > minCount) {
-                if (excluded == null || !excluded.contains(value)) {
-                    InternalFloatTermsFacet.FloatEntry entry = new InternalFloatTermsFacet.FloatEntry(value, count);
-                    ordered.add(entry);
-                }
-            }
-        }
-
-        for (ReaderAggregator aggregator : aggregators) {
-            CacheRecycler.pushIntArray(aggregator.counts);
-        }
-
-        return new InternalFloatTermsFacet(facetName, comparatorType, size, ordered, missing, total);
-    }
-
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
-
-        final float[] values;
-        final int[] counts;
-
-        int position = 0;
-        float current;
-        int total;
-
-        public ReaderAggregator(FloatFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
-        }
-
-        @Override
-        public void onOrdinal(int docId, int ordinal) {
-            counts[ordinal]++;
-            total++;
-        }
-
-        public boolean nextPosition() {
-            if (++position >= values.length) {
-                return false;
-            }
-            current = values[position];
-            return true;
-        }
-    }
-
-    public static class AggregatorPriorityQueue extends PriorityQueue<ReaderAggregator> {
-
-        public AggregatorPriorityQueue(int size) {
-            super(size);
-        }
-
-        @Override
-        protected boolean lessThan(ReaderAggregator a, ReaderAggregator b) {
-            return a.current < b.current;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java
deleted file mode 100644
index 644dc6acf40..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.ints;
-
-import com.google.common.collect.ImmutableList;
-import gnu.trove.iterator.TIntIntIterator;
-import gnu.trove.map.hash.TIntIntHashMap;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.text.StringText;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.InternalTermsFacet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class InternalIntTermsFacet extends InternalTermsFacet {
-
-    private static final String STREAM_TYPE = "iTerms";
-
-    public static void registerStream() {
-        Streams.registerStream(STREAM, STREAM_TYPE);
-    }
-
-    static Stream STREAM = new Stream() {
-        @Override
-        public Facet readFacet(String type, StreamInput in) throws IOException {
-            return readTermsFacet(in);
-        }
-    };
-
-    @Override
-    public String streamType() {
-        return STREAM_TYPE;
-    }
-
-    public static class IntEntry implements Entry {
-
-        int term;
-        int count;
-
-        public IntEntry(int term, int count) {
-            this.term = term;
-            this.count = count;
-        }
-
-        public Text term() {
-            return new StringText(Integer.toString(term));
-        }
-
-        public Text getTerm() {
-            return term();
-        }
-
-        @Override
-        public Number termAsNumber() {
-            return term;
-        }
-
-        @Override
-        public Number getTermAsNumber() {
-            return termAsNumber();
-        }
-
-        public int count() {
-            return count;
-        }
-
-        public int getCount() {
-            return count();
-        }
-
-        @Override
-        public int compareTo(Entry o) {
-            int anotherVal = ((IntEntry) o).term;
-            int i = term - anotherVal;
-            if (i == 0) {
-                i = count - o.count();
-                if (i == 0) {
-                    i = System.identityHashCode(this) - System.identityHashCode(o);
-                }
-            }
-            return i;
-        }
-    }
-
-    private String name;
-
-    int requiredSize;
-
-    long missing;
-    long total;
-
-    Collection<IntEntry> entries = ImmutableList.of();
-
-    ComparatorType comparatorType;
-
-    InternalIntTermsFacet() {
-    }
-
-    public InternalIntTermsFacet(String name, ComparatorType comparatorType, int requiredSize, Collection<IntEntry> entries, long missing, long total) {
-        this.name = name;
-        this.comparatorType = comparatorType;
-        this.requiredSize = requiredSize;
-        this.entries = entries;
-        this.missing = missing;
-        this.total = total;
-    }
-
-    @Override
-    public String name() {
-        return this.name;
-    }
-
-    @Override
-    public String getName() {
-        return this.name;
-    }
-
-    @Override
-    public String type() {
-        return TYPE;
-    }
-
-    @Override
-    public String getType() {
-        return type();
-    }
-
-    @Override
-    public List<IntEntry> entries() {
-        if (!(entries instanceof List)) {
-            entries = ImmutableList.copyOf(entries);
-        }
-        return (List<IntEntry>) entries;
-    }
-
-    @Override
-    public List<IntEntry> getEntries() {
-        return entries();
-    }
-
-    @SuppressWarnings({"unchecked"})
-    @Override
-    public Iterator<Entry> iterator() {
-        return (Iterator) entries.iterator();
-    }
-
-    @Override
-    public long missingCount() {
-        return this.missing;
-    }
-
-    @Override
-    public long getMissingCount() {
-        return missingCount();
-    }
-
-    @Override
-    public long totalCount() {
-        return this.total;
-    }
-
-    @Override
-    public long getTotalCount() {
-        return totalCount();
-    }
-
-    @Override
-    public long otherCount() {
-        long other = total;
-        for (Entry entry : entries) {
-            other -= entry.count();
-        }
-        return other;
-    }
-
-    @Override
-    public long getOtherCount() {
-        return otherCount();
-    }
-
-    @Override
-    public Facet reduce(String name, List<Facet> facets) {
-        if (facets.size() == 1) {
-            return facets.get(0);
-        }
-        InternalIntTermsFacet first = (InternalIntTermsFacet) facets.get(0);
-        TIntIntHashMap aggregated = CacheRecycler.popIntIntMap();
-        long missing = 0;
-        long total = 0;
-        for (Facet facet : facets) {
-            InternalIntTermsFacet mFacet = (InternalIntTermsFacet) facet;
-            missing += mFacet.missingCount();
-            total += mFacet.totalCount();
-            for (IntEntry entry : mFacet.entries) {
-                aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
-            }
-        }
-
-        BoundedTreeSet<IntEntry> ordered = new BoundedTreeSet<IntEntry>(first.comparatorType.comparator(), first.requiredSize);
-        for (TIntIntIterator it = aggregated.iterator(); it.hasNext(); ) {
-            it.advance();
-            ordered.add(new IntEntry(it.key(), it.value()));
-        }
-        first.entries = ordered;
-        first.missing = missing;
-        first.total = total;
-
-        CacheRecycler.pushIntIntMap(aggregated);
-
-        return first;
-    }
-
-    static final class Fields {
-        static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
-        static final XContentBuilderString MISSING = new XContentBuilderString("missing");
-        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-        static final XContentBuilderString OTHER = new XContentBuilderString("other");
-        static final XContentBuilderString TERMS = new XContentBuilderString("terms");
-        static final XContentBuilderString TERM = new XContentBuilderString("term");
-        static final XContentBuilderString COUNT = new XContentBuilderString("count");
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(name);
-        builder.field(Fields._TYPE, TermsFacet.TYPE);
-        builder.field(Fields.MISSING, missing);
-        builder.field(Fields.TOTAL, total);
-        builder.field(Fields.OTHER, otherCount());
-        builder.startArray(Fields.TERMS);
-        for (IntEntry entry : entries) {
-            builder.startObject();
-            builder.field(Fields.TERM, entry.term);
-            builder.field(Fields.COUNT, entry.count());
-            builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-    public static InternalIntTermsFacet readTermsFacet(StreamInput in) throws IOException {
-        InternalIntTermsFacet facet = new InternalIntTermsFacet();
-        facet.readFrom(in);
-        return facet;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        comparatorType = ComparatorType.fromId(in.readByte());
-        requiredSize = in.readVInt();
-        missing = in.readVLong();
-        total = in.readVLong();
-
-        int size = in.readVInt();
-        entries = new ArrayList<IntEntry>(size);
-        for (int i = 0; i < size; i++) {
-            entries.add(new IntEntry(in.readInt(), in.readVInt()));
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeByte(comparatorType.id());
-        out.writeVInt(requiredSize);
-        out.writeVLong(missing);
-        out.writeVLong(total);
-
-        out.writeVInt(entries.size());
-        for (IntEntry entry : entries) {
-            out.writeInt(entry.term);
-            out.writeVInt(entry.count());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java
deleted file mode 100644
index 0cadda36c12..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntFacetCollector.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.ints;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.iterator.TIntIntIterator;
-import gnu.trove.map.hash.TIntIntHashMap;
-import gnu.trove.set.hash.TIntHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.ints.IntFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Set;
-
-/**
- *
- */
-public class TermsIntFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final FieldDataType fieldDataType;
-
-    private IntFieldData fieldData;
-
-    private final StaticAggregatorValueProc aggregator;
-
-    private final SearchScript script;
-
-    public TermsIntFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                  ImmutableSet<BytesRef> excluded, String scriptLang, String script, Map<String, Object> params) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms int facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.INT) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of int type, can't run terms int facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
-
-        if (this.script == null && excluded.isEmpty()) {
-            aggregator = new StaticAggregatorValueProc(CacheRecycler.popIntIntMap());
-        } else {
-            aggregator = new AggregatorValueProc(CacheRecycler.popIntIntMap(), excluded, this.script);
-        }
-
-        if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    IntFieldData fieldData = (IntFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
-        }
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) throws IOException {
-        if (script != null) {
-            script.setScorer(scorer);
-        }
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (IntFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        if (script != null) {
-            script.setNextReader(context);
-        }
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
-    }
-
-    @Override
-    public Facet facet() {
-        TIntIntHashMap facets = aggregator.facets();
-        if (facets.isEmpty()) {
-            CacheRecycler.pushIntIntMap(facets);
-            return new InternalIntTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalIntTermsFacet.IntEntry>of(), aggregator.missing(), aggregator.total());
-        } else {
-            if (size < EntryPriorityQueue.LIMIT) {
-                EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-                for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.insertWithOverflow(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
-                }
-                InternalIntTermsFacet.IntEntry[] list = new InternalIntTermsFacet.IntEntry[ordered.size()];
-                for (int i = ordered.size() - 1; i >= 0; i--) {
-                    list[i] = (InternalIntTermsFacet.IntEntry) ordered.pop();
-                }
-                CacheRecycler.pushIntIntMap(facets);
-                return new InternalIntTermsFacet(facetName, comparatorType, size, Arrays.asList(list), aggregator.missing(), aggregator.total());
-            } else {
-                BoundedTreeSet<InternalIntTermsFacet.IntEntry> ordered = new BoundedTreeSet<InternalIntTermsFacet.IntEntry>(comparatorType.comparator(), size);
-                for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.add(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
-                }
-                CacheRecycler.pushIntIntMap(facets);
-                return new InternalIntTermsFacet(facetName, comparatorType, size, ordered, aggregator.missing(), aggregator.total());
-            }
-        }
-    }
-
-    public static class AggregatorValueProc extends StaticAggregatorValueProc {
-
-        private final SearchScript script;
-
-        private final TIntHashSet excluded;
-
-        public AggregatorValueProc(TIntIntHashMap facets, Set<BytesRef> excluded, SearchScript script) {
-            super(facets);
-            if (excluded == null || excluded.isEmpty()) {
-                this.excluded = null;
-            } else {
-                this.excluded = new TIntHashSet(excluded.size());
-                for (BytesRef s : excluded) {
-                    this.excluded.add(Integer.parseInt(s.utf8ToString()));
-                }
-            }
-            this.script = script;
-        }
-
-        @Override
-        public void onValue(int docId, int value) {
-            if (excluded != null && excluded.contains(value)) {
-                return;
-            }
-            if (script != null) {
-                script.setNextDocId(docId);
-                script.setNextVar("term", value);
-                Object scriptValue = script.run();
-                if (scriptValue == null) {
-                    return;
-                }
-                if (scriptValue instanceof Boolean) {
-                    if (!((Boolean) scriptValue)) {
-                        return;
-                    }
-                } else {
-                    value = ((Number) scriptValue).intValue();
-                }
-            }
-            super.onValue(docId, value);
-        }
-    }
-
-    public static class StaticAggregatorValueProc implements IntFieldData.ValueInDocProc, IntFieldData.ValueProc {
-
-        private final TIntIntHashMap facets;
-
-        private int missing;
-        private int total;
-
-        public StaticAggregatorValueProc(TIntIntHashMap facets) {
-            this.facets = facets;
-        }
-
-        @Override
-        public void onValue(int value) {
-            facets.putIfAbsent(value, 0);
-        }
-
-        @Override
-        public void onValue(int docId, int value) {
-            facets.adjustOrPutValue(value, 1, 1);
-            total++;
-        }
-
-        @Override
-        public void onMissing(int docId) {
-            missing++;
-        }
-
-        public final TIntIntHashMap facets() {
-            return facets;
-        }
-
-        public final int missing() {
-            return this.missing;
-        }
-
-        public final int total() {
-            return this.total;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntOrdinalsFacetCollector.java
deleted file mode 100644
index 9ba28768bff..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/ints/TermsIntOrdinalsFacetCollector.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.ints;
-
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.set.hash.TIntHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.ints.IntFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- *
- */
-public class TermsIntOrdinalsFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final int minCount;
-
-    private final FieldDataType fieldDataType;
-
-    private IntFieldData fieldData;
-
-    private final List<ReaderAggregator> aggregators;
-
-    private ReaderAggregator current;
-
-    long missing;
-    long total;
-
-    private final TIntHashSet excluded;
-
-    public TermsIntOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                          ImmutableSet<BytesRef> excluded) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms int facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.INT) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of int type, can't run terms int facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (excluded == null || excluded.isEmpty()) {
-            this.excluded = null;
-        } else {
-            this.excluded = new TIntHashSet(excluded.size());
-            for (BytesRef s : excluded) {
-                this.excluded.add(Integer.parseInt(s.utf8ToString()));
-            }
-        }
-
-        // minCount is offset by -1
-        if (allTerms) {
-            minCount = -1;
-        } else {
-            minCount = 0;
-        }
-
-        this.aggregators = new ArrayList<ReaderAggregator>(context.searcher().getIndexReader().leaves().size());
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-        fieldData = (IntFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
-    }
-
-    @Override
-    public Facet facet() {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            // if we have values for this one, add it
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-
-        AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
-
-        for (ReaderAggregator aggregator : aggregators) {
-            if (aggregator.nextPosition()) {
-                queue.add(aggregator);
-            }
-        }
-
-        // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
-        if (size < EntryPriorityQueue.LIMIT) {
-            // optimize to use priority size
-            EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-
-            while (queue.size() > 0) {
-                ReaderAggregator agg = queue.top();
-                int value = agg.current;
-                int count = 0;
-                do {
-                    count += agg.counts[agg.position];
-                    if (agg.nextPosition()) {
-                        agg = queue.updateTop();
-                    } else {
-                        // we are done with this reader
-                        queue.pop();
-                        agg = queue.top();
-                    }
-                } while (agg != null && value == agg.current);
-
-                if (count > minCount) {
-                    if (excluded == null || !excluded.contains(value)) {
-                        InternalIntTermsFacet.IntEntry entry = new InternalIntTermsFacet.IntEntry(value, count);
-                        ordered.insertWithOverflow(entry);
-                    }
-                }
-            }
-            InternalIntTermsFacet.IntEntry[] list = new InternalIntTermsFacet.IntEntry[ordered.size()];
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = (InternalIntTermsFacet.IntEntry) ordered.pop();
-            }
-
-            for (ReaderAggregator aggregator : aggregators) {
-                CacheRecycler.pushIntArray(aggregator.counts);
-            }
-
-            return new InternalIntTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
-        }
-
-        BoundedTreeSet<InternalIntTermsFacet.IntEntry> ordered = new BoundedTreeSet<InternalIntTermsFacet.IntEntry>(comparatorType.comparator(), size);
-
-        while (queue.size() > 0) {
-            ReaderAggregator agg = queue.top();
-            int value = agg.current;
-            int count = 0;
-            do {
-                count += agg.counts[agg.position];
-                if (agg.nextPosition()) {
-                    agg = queue.updateTop();
-                } else {
-                    // we are done with this reader
-                    queue.pop();
-                    agg = queue.top();
-                }
-            } while (agg != null && value == agg.current);
-
-            if (count > minCount) {
-                if (excluded == null || !excluded.contains(value)) {
-                    InternalIntTermsFacet.IntEntry entry = new InternalIntTermsFacet.IntEntry(value, count);
-                    ordered.add(entry);
-                }
-            }
-        }
-
-        for (ReaderAggregator aggregator : aggregators) {
-            CacheRecycler.pushIntArray(aggregator.counts);
-        }
-
-        return new InternalIntTermsFacet(facetName, comparatorType, size, ordered, missing, total);
-    }
-
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
-
-        final int[] values;
-        final int[] counts;
-
-        int position = 0;
-        int current;
-        int total = 0;
-
-        public ReaderAggregator(IntFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
-        }
-
-        @Override
-        public void onOrdinal(int docId, int ordinal) {
-            counts[ordinal]++;
-            total++;
-        }
-
-        public boolean nextPosition() {
-            if (++position >= values.length) {
-                return false;
-            }
-            current = values[position];
-            return true;
-        }
-    }
-
-    public static class AggregatorPriorityQueue extends PriorityQueue<ReaderAggregator> {
-
-        public AggregatorPriorityQueue(int size) {
-            super(size);
-        }
-
-        @Override
-        protected boolean lessThan(ReaderAggregator a, ReaderAggregator b) {
-            return a.current < b.current;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/ip/InternalIpTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/ip/InternalIpTermsFacet.java
deleted file mode 100644
index 7fb5dcb92aa..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/ip/InternalIpTermsFacet.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.ip;
-
-import com.google.common.collect.ImmutableList;
-import gnu.trove.iterator.TLongIntIterator;
-import gnu.trove.map.hash.TLongIntHashMap;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.text.StringText;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.index.mapper.ip.IpFieldMapper;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.InternalTermsFacet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class InternalIpTermsFacet extends InternalTermsFacet {
-
-    private static final String STREAM_TYPE = "ipTerms";
-
-    public static void registerStream() {
-        Streams.registerStream(STREAM, STREAM_TYPE);
-    }
-
-    static Stream STREAM = new Stream() {
-        @Override
-        public Facet readFacet(String type, StreamInput in) throws IOException {
-            return readTermsFacet(in);
-        }
-    };
-
-    @Override
-    public String streamType() {
-        return STREAM_TYPE;
-    }
-
-    public static class LongEntry implements Entry {
-
-        long term;
-        int count;
-
-        public LongEntry(long term, int count) {
-            this.term = term;
-            this.count = count;
-        }
-
-        public Text term() {
-            return new StringText(IpFieldMapper.longToIp(term));
-        }
-
-        public Text getTerm() {
-            return term();
-        }
-
-        @Override
-        public Number termAsNumber() {
-            return term;
-        }
-
-        @Override
-        public Number getTermAsNumber() {
-            return termAsNumber();
-        }
-
-        public int count() {
-            return count;
-        }
-
-        public int getCount() {
-            return count();
-        }
-
-        @Override
-        public int compareTo(Entry o) {
-            long anotherVal = ((LongEntry) o).term;
-            if (term < anotherVal) {
-                return -1;
-            }
-            if (term == anotherVal) {
-                int i = count - o.count();
-                if (i == 0) {
-                    i = System.identityHashCode(this) - System.identityHashCode(o);
-                }
-                return i;
-            }
-            return 1;
-        }
-    }
-
-    private String name;
-
-    int requiredSize;
-
-    long missing;
-    long total;
-
-    Collection<LongEntry> entries = ImmutableList.of();
-
-    ComparatorType comparatorType;
-
-    InternalIpTermsFacet() {
-    }
-
-    public InternalIpTermsFacet(String name, ComparatorType comparatorType, int requiredSize, Collection<LongEntry> entries, long missing, long total) {
-        this.name = name;
-        this.comparatorType = comparatorType;
-        this.requiredSize = requiredSize;
-        this.entries = entries;
-        this.missing = missing;
-        this.total = total;
-    }
-
-    @Override
-    public String name() {
-        return this.name;
-    }
-
-    @Override
-    public String getName() {
-        return this.name;
-    }
-
-    @Override
-    public String type() {
-        return TYPE;
-    }
-
-    @Override
-    public String getType() {
-        return type();
-    }
-
-    @Override
-    public List<LongEntry> entries() {
-        if (!(entries instanceof List)) {
-            entries = ImmutableList.copyOf(entries);
-        }
-        return (List<LongEntry>) entries;
-    }
-
-    @Override
-    public List<LongEntry> getEntries() {
-        return entries();
-    }
-
-    @SuppressWarnings({"unchecked"})
-    @Override
-    public Iterator<Entry> iterator() {
-        return (Iterator) entries.iterator();
-    }
-
-    @Override
-    public long missingCount() {
-        return this.missing;
-    }
-
-    @Override
-    public long getMissingCount() {
-        return missingCount();
-    }
-
-
-    @Override
-    public long totalCount() {
-        return this.total;
-    }
-
-    @Override
-    public long getTotalCount() {
-        return totalCount();
-    }
-
-    @Override
-    public long otherCount() {
-        long other = total;
-        for (Entry entry : entries) {
-            other -= entry.count();
-        }
-        return other;
-    }
-
-    @Override
-    public long getOtherCount() {
-        return otherCount();
-    }
-
-    @Override
-    public Facet reduce(String name, List<Facet> facets) {
-        if (facets.size() == 1) {
-            return facets.get(0);
-        }
-        InternalIpTermsFacet first = (InternalIpTermsFacet) facets.get(0);
-        TLongIntHashMap aggregated = CacheRecycler.popLongIntMap();
-        long missing = 0;
-        long total = 0;
-        for (Facet facet : facets) {
-            InternalIpTermsFacet mFacet = (InternalIpTermsFacet) facet;
-            missing += mFacet.missingCount();
-            total += mFacet.totalCount();
-            for (LongEntry entry : mFacet.entries) {
-                aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
-            }
-        }
-
-        BoundedTreeSet<LongEntry> ordered = new BoundedTreeSet<LongEntry>(first.comparatorType.comparator(), first.requiredSize);
-        for (TLongIntIterator it = aggregated.iterator(); it.hasNext(); ) {
-            it.advance();
-            ordered.add(new LongEntry(it.key(), it.value()));
-        }
-        first.entries = ordered;
-        first.missing = missing;
-        first.total = total;
-
-        CacheRecycler.pushLongIntMap(aggregated);
-
-        return first;
-    }
-
-    static final class Fields {
-        static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
-        static final XContentBuilderString MISSING = new XContentBuilderString("missing");
-        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-        static final XContentBuilderString OTHER = new XContentBuilderString("other");
-        static final XContentBuilderString TERMS = new XContentBuilderString("terms");
-        static final XContentBuilderString TERM = new XContentBuilderString("term");
-        static final XContentBuilderString COUNT = new XContentBuilderString("count");
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(name);
-        builder.field(Fields._TYPE, TermsFacet.TYPE);
-        builder.field(Fields.MISSING, missing);
-        builder.field(Fields.TOTAL, total);
-        builder.field(Fields.OTHER, otherCount());
-        builder.startArray(Fields.TERMS);
-        for (LongEntry entry : entries) {
-            builder.startObject();
-            builder.field(Fields.TERM, entry.term()); // displayed as string
-            builder.field(Fields.COUNT, entry.count());
-            builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-    public static InternalIpTermsFacet readTermsFacet(StreamInput in) throws IOException {
-        InternalIpTermsFacet facet = new InternalIpTermsFacet();
-        facet.readFrom(in);
-        return facet;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        comparatorType = ComparatorType.fromId(in.readByte());
-        requiredSize = in.readVInt();
-        missing = in.readVLong();
-        total = in.readVLong();
-
-        int size = in.readVInt();
-        entries = new ArrayList<LongEntry>(size);
-        for (int i = 0; i < size; i++) {
-            entries.add(new LongEntry(in.readLong(), in.readVInt()));
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeByte(comparatorType.id());
-        out.writeVInt(requiredSize);
-        out.writeVLong(missing);
-        out.writeVLong(total);
-
-        out.writeVInt(entries.size());
-        for (LongEntry entry : entries) {
-            out.writeLong(entry.term);
-            out.writeVInt(entry.count());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpFacetCollector.java
deleted file mode 100644
index ca5c46e6703..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpFacetCollector.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.ip;
-
-import com.google.common.collect.ImmutableList;
-import gnu.trove.iterator.TLongIntIterator;
-import gnu.trove.map.hash.TLongIntHashMap;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.Scorer;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-
-/**
- *
- */
-public class TermsIpFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final FieldDataType fieldDataType;
-
-    private LongFieldData fieldData;
-
-    private final StaticAggregatorValueProc aggregator;
-
-    private final SearchScript script;
-
-    public TermsIpFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                 String scriptLang, String script, Map<String, Object> params) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms long facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.LONG) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of long type, can't run terms long facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
-
-        if (this.script == null) {
-            aggregator = new StaticAggregatorValueProc(CacheRecycler.popLongIntMap());
-        } else {
-            aggregator = new AggregatorValueProc(CacheRecycler.popLongIntMap(), this.script);
-        }
-
-        if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    LongFieldData fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
-        }
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) throws IOException {
-        if (script != null) {
-            script.setScorer(scorer);
-        }
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        if (script != null) {
-            script.setNextReader(context);
-        }
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
-    }
-
-    @Override
-    public Facet facet() {
-        TLongIntHashMap facets = aggregator.facets();
-        if (facets.isEmpty()) {
-            CacheRecycler.pushLongIntMap(facets);
-            return new InternalIpTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalIpTermsFacet.LongEntry>of(), aggregator.missing(), aggregator.total());
-        } else {
-            if (size < EntryPriorityQueue.LIMIT) {
-                EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-                for (TLongIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.insertWithOverflow(new InternalIpTermsFacet.LongEntry(it.key(), it.value()));
-                }
-                InternalIpTermsFacet.LongEntry[] list = new InternalIpTermsFacet.LongEntry[ordered.size()];
-                for (int i = ordered.size() - 1; i >= 0; i--) {
-                    list[i] = (InternalIpTermsFacet.LongEntry) ordered.pop();
-                }
-                CacheRecycler.pushLongIntMap(facets);
-                return new InternalIpTermsFacet(facetName, comparatorType, size, Arrays.asList(list), aggregator.missing(), aggregator.total());
-            } else {
-                BoundedTreeSet<InternalIpTermsFacet.LongEntry> ordered = new BoundedTreeSet<InternalIpTermsFacet.LongEntry>(comparatorType.comparator(), size);
-                for (TLongIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.add(new InternalIpTermsFacet.LongEntry(it.key(), it.value()));
-                }
-                CacheRecycler.pushLongIntMap(facets);
-                return new InternalIpTermsFacet(facetName, comparatorType, size, ordered, aggregator.missing(), aggregator.total());
-            }
-        }
-    }
-
-    public static class AggregatorValueProc extends StaticAggregatorValueProc {
-
-        private final SearchScript script;
-
-        public AggregatorValueProc(TLongIntHashMap facets, SearchScript script) {
-            super(facets);
-            this.script = script;
-        }
-
-        @Override
-        public void onValue(int docId, long value) {
-            if (script != null) {
-                script.setNextDocId(docId);
-                script.setNextVar("term", value);
-                Object scriptValue = script.run();
-                if (scriptValue == null) {
-                    return;
-                }
-                if (scriptValue instanceof Boolean) {
-                    if (!((Boolean) scriptValue)) {
-                        return;
-                    }
-                } else {
-                    value = ((Number) scriptValue).longValue();
-                }
-            }
-            super.onValue(docId, value);
-        }
-    }
-
-    public static class StaticAggregatorValueProc implements LongFieldData.ValueInDocProc, LongFieldData.ValueProc {
-
-        private final TLongIntHashMap facets;
-
-        private int missing;
-        private int total;
-
-        public StaticAggregatorValueProc(TLongIntHashMap facets) {
-            this.facets = facets;
-        }
-
-        @Override
-        public void onValue(long value) {
-            facets.putIfAbsent(value, 0);
-        }
-
-        @Override
-        public void onValue(int docId, long value) {
-            facets.adjustOrPutValue(value, 1, 1);
-            total++;
-        }
-
-        @Override
-        public void onMissing(int docId) {
-            missing++;
-        }
-
-        public final TLongIntHashMap facets() {
-            return facets;
-        }
-
-        public final int missing() {
-            return this.missing;
-        }
-
-        public final int total() {
-            return this.total;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpOrdinalsFacetCollector.java
deleted file mode 100644
index 9e6a5a594ce..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/ip/TermsIpOrdinalsFacetCollector.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.ip;
-
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.set.hash.TLongHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- *
- */
-public class TermsIpOrdinalsFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final int minCount;
-
-    private final FieldDataType fieldDataType;
-
-    private LongFieldData fieldData;
-
-    private final List<ReaderAggregator> aggregators;
-
-    private ReaderAggregator current;
-
-    long missing;
-    long total;
-
-    private final TLongHashSet excluded;
-
-    public TermsIpOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                         ImmutableSet<String> excluded) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms long facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.LONG) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of long type, can't run terms long facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (excluded == null || excluded.isEmpty()) {
-            this.excluded = null;
-        } else {
-            this.excluded = new TLongHashSet(excluded.size());
-            for (String s : excluded) {
-                this.excluded.add(Long.parseLong(s));
-            }
-        }
-
-        // minCount is offset by -1
-        if (allTerms) {
-            minCount = -1;
-        } else {
-            minCount = 0;
-        }
-
-        this.aggregators = new ArrayList<ReaderAggregator>(context.searcher().getIndexReader().leaves().size());
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-        fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
-    }
-
-    @Override
-    public Facet facet() {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            // if we have values for this one, add it
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-
-        AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
-
-        for (ReaderAggregator aggregator : aggregators) {
-            if (aggregator.nextPosition()) {
-                queue.add(aggregator);
-            }
-        }
-
-        // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
-        if (size < EntryPriorityQueue.LIMIT) {
-            // optimize to use priority size
-            EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-
-            while (queue.size() > 0) {
-                ReaderAggregator agg = queue.top();
-                long value = agg.current;
-                int count = 0;
-                do {
-                    count += agg.counts[agg.position];
-                    if (agg.nextPosition()) {
-                        agg = queue.updateTop();
-                    } else {
-                        // we are done with this reader
-                        queue.pop();
-                        agg = queue.top();
-                    }
-                } while (agg != null && value == agg.current);
-
-                if (count > minCount) {
-                    if (excluded == null || !excluded.contains(value)) {
-                        InternalIpTermsFacet.LongEntry entry = new InternalIpTermsFacet.LongEntry(value, count);
-                        ordered.insertWithOverflow(entry);
-                    }
-                }
-            }
-            InternalIpTermsFacet.LongEntry[] list = new InternalIpTermsFacet.LongEntry[ordered.size()];
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = (InternalIpTermsFacet.LongEntry) ordered.pop();
-            }
-
-            for (ReaderAggregator aggregator : aggregators) {
-                CacheRecycler.pushIntArray(aggregator.counts);
-            }
-
-            return new InternalIpTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
-        }
-
-        BoundedTreeSet<InternalIpTermsFacet.LongEntry> ordered = new BoundedTreeSet<InternalIpTermsFacet.LongEntry>(comparatorType.comparator(), size);
-
-        while (queue.size() > 0) {
-            ReaderAggregator agg = queue.top();
-            long value = agg.current;
-            int count = 0;
-            do {
-                count += agg.counts[agg.position];
-                if (agg.nextPosition()) {
-                    agg = queue.updateTop();
-                } else {
-                    // we are done with this reader
-                    queue.pop();
-                    agg = queue.top();
-                }
-            } while (agg != null && value == agg.current);
-
-            if (count > minCount) {
-                if (excluded == null || !excluded.contains(value)) {
-                    InternalIpTermsFacet.LongEntry entry = new InternalIpTermsFacet.LongEntry(value, count);
-                    ordered.add(entry);
-                }
-            }
-        }
-
-        for (ReaderAggregator aggregator : aggregators) {
-            CacheRecycler.pushIntArray(aggregator.counts);
-        }
-
-        return new InternalIpTermsFacet(facetName, comparatorType, size, ordered, missing, total);
-    }
-
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
-
-        final long[] values;
-        final int[] counts;
-
-        int position = 0;
-        long current = Integer.MIN_VALUE;
-        int total;
-
-        public ReaderAggregator(LongFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
-        }
-
-        @Override
-        public void onOrdinal(int docId, int ordinal) {
-            counts[ordinal]++;
-            total++;
-        }
-
-        public boolean nextPosition() {
-            if (++position >= values.length) {
-                return false;
-            }
-            current = values[position];
-            return true;
-        }
-    }
-
-    public static class AggregatorPriorityQueue extends PriorityQueue<ReaderAggregator> {
-
-        public AggregatorPriorityQueue(int size) {
-            super(size);
-        }
-
-        @Override
-        protected boolean lessThan(ReaderAggregator a, ReaderAggregator b) {
-            return a.current < b.current;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java
index 7b17f60c606..929ade5d3f6 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongFacetCollector.java
@@ -24,46 +24,30 @@ import com.google.common.collect.ImmutableSet;
 import gnu.trove.iterator.TLongIntIterator;
 import gnu.trove.map.hash.TLongIntHashMap;
 import gnu.trove.set.hash.TLongHashSet;
-import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.LongValues;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.terms.TermsFacet;
 import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Set;
 
 /**
  *
  */
 public class TermsLongFacetCollector extends AbstractFacetCollector {
 
-    static ThreadLocal<ThreadLocals.CleanableValue<Deque<TLongIntHashMap>>> cache = new ThreadLocal<ThreadLocals.CleanableValue<Deque<TLongIntHashMap>>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<Deque<TLongIntHashMap>> initialValue() {
-            return new ThreadLocals.CleanableValue<Deque<TLongIntHashMap>>(new ArrayDeque<TLongIntHashMap>());
-        }
-    };
-
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
+    private final IndexNumericFieldData indexFieldData;
 
     private final TermsFacet.ComparatorType comparatorType;
 
@@ -71,43 +55,20 @@ public class TermsLongFacetCollector extends AbstractFacetCollector {
 
     private final int numberOfShards;
 
-    private final FieldDataType fieldDataType;
-
-    private LongFieldData fieldData;
+    private LongValues values;
 
     private final StaticAggregatorValueProc aggregator;
 
     private final SearchScript script;
 
-    public TermsLongFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                   ImmutableSet<BytesRef> excluded, String scriptLang, String script, Map<String, Object> params) {
+    public TermsLongFacetCollector(String facetName, IndexNumericFieldData indexFieldData, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
+                                   ImmutableSet<BytesRef> excluded, SearchScript script) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
+        this.indexFieldData = indexFieldData;
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms long facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.LONG) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of long type, can't run terms long facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
+        this.script = script;
 
         if (this.script == null && excluded.isEmpty()) {
             aggregator = new StaticAggregatorValueProc(CacheRecycler.popLongIntMap());
@@ -115,16 +76,17 @@ public class TermsLongFacetCollector extends AbstractFacetCollector {
             aggregator = new AggregatorValueProc(CacheRecycler.popLongIntMap(), excluded, this.script);
         }
 
-        if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    LongFieldData fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
-        }
+        // TODO: we need to support this with the new field data....
+//        if (allTerms) {
+//            try {
+//                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
+//                    LongFieldData fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
+//                    fieldData.forEachValue(aggregator);
+//                }
+//            } catch (Exception e) {
+//                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
+//            }
+//        }
     }
 
     @Override
@@ -136,7 +98,7 @@ public class TermsLongFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getLongValues();
         if (script != null) {
             script.setNextReader(context);
         }
@@ -144,7 +106,7 @@ public class TermsLongFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
+        values.forEachValueInDoc(doc, aggregator);
     }
 
     @Override
@@ -221,7 +183,7 @@ public class TermsLongFacetCollector extends AbstractFacetCollector {
         }
     }
 
-    public static class StaticAggregatorValueProc implements LongFieldData.ValueInDocProc, LongFieldData.ValueProc {
+    public static class StaticAggregatorValueProc implements LongValues.ValueInDocProc {
 
         private final TLongIntHashMap facets;
 
@@ -232,11 +194,6 @@ public class TermsLongFacetCollector extends AbstractFacetCollector {
             this.facets = facets;
         }
 
-        @Override
-        public void onValue(long value) {
-            facets.putIfAbsent(value, 0);
-        }
-
         @Override
         public void onValue(int docId, long value) {
             facets.adjustOrPutValue(value, 1, 1);
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongOrdinalsFacetCollector.java
deleted file mode 100644
index 4fd7f15b7f9..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/longs/TermsLongOrdinalsFacetCollector.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.longs;
-
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.set.hash.TLongHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- *
- */
-public class TermsLongOrdinalsFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final int minCount;
-
-    private final FieldDataType fieldDataType;
-
-    private LongFieldData fieldData;
-
-    private final List<ReaderAggregator> aggregators;
-
-    private ReaderAggregator current;
-
-    long missing;
-    long total;
-
-    private final TLongHashSet excluded;
-
-    public TermsLongOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                           ImmutableSet<BytesRef> excluded) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms long facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.LONG) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of long type, can't run terms long facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (excluded == null || excluded.isEmpty()) {
-            this.excluded = null;
-        } else {
-            this.excluded = new TLongHashSet(excluded.size());
-            for (BytesRef s : excluded) {
-                this.excluded.add(Long.parseLong(s.utf8ToString()));
-            }
-        }
-
-        // minCount is offset by -1
-        if (allTerms) {
-            minCount = -1;
-        } else {
-            minCount = 0;
-        }
-
-        this.aggregators = new ArrayList<ReaderAggregator>(context.searcher().getIndexReader().leaves().size());
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-        fieldData = (LongFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
-    }
-
-    @Override
-    public Facet facet() {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            // if we have values for this one, add it
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-
-        AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
-
-        for (ReaderAggregator aggregator : aggregators) {
-            if (aggregator.nextPosition()) {
-                queue.add(aggregator);
-            }
-        }
-
-        // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
-        if (size < EntryPriorityQueue.LIMIT) {
-            // optimize to use priority size
-            EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-
-            while (queue.size() > 0) {
-                ReaderAggregator agg = queue.top();
-                long value = agg.current;
-                int count = 0;
-                do {
-                    count += agg.counts[agg.position];
-                    if (agg.nextPosition()) {
-                        agg = queue.updateTop();
-                    } else {
-                        // we are done with this reader
-                        queue.pop();
-                        agg = queue.top();
-                    }
-                } while (agg != null && value == agg.current);
-
-                if (count > minCount) {
-                    if (excluded == null || !excluded.contains(value)) {
-                        InternalLongTermsFacet.LongEntry entry = new InternalLongTermsFacet.LongEntry(value, count);
-                        ordered.insertWithOverflow(entry);
-                    }
-                }
-            }
-            InternalLongTermsFacet.LongEntry[] list = new InternalLongTermsFacet.LongEntry[ordered.size()];
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = (InternalLongTermsFacet.LongEntry) ordered.pop();
-            }
-
-            for (ReaderAggregator aggregator : aggregators) {
-                CacheRecycler.pushIntArray(aggregator.counts);
-            }
-
-            return new InternalLongTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
-        }
-
-        BoundedTreeSet<InternalLongTermsFacet.LongEntry> ordered = new BoundedTreeSet<InternalLongTermsFacet.LongEntry>(comparatorType.comparator(), size);
-
-        while (queue.size() > 0) {
-            ReaderAggregator agg = queue.top();
-            long value = agg.current;
-            int count = 0;
-            do {
-                count += agg.counts[agg.position];
-                if (agg.nextPosition()) {
-                    agg = queue.updateTop();
-                } else {
-                    // we are done with this reader
-                    queue.pop();
-                    agg = queue.top();
-                }
-            } while (agg != null && value == agg.current);
-
-            if (count > minCount) {
-                if (excluded == null || !excluded.contains(value)) {
-                    InternalLongTermsFacet.LongEntry entry = new InternalLongTermsFacet.LongEntry(value, count);
-                    ordered.add(entry);
-                }
-            }
-        }
-
-        for (ReaderAggregator aggregator : aggregators) {
-            CacheRecycler.pushIntArray(aggregator.counts);
-        }
-
-        return new InternalLongTermsFacet(facetName, comparatorType, size, ordered, missing, total);
-    }
-
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
-
-        final long[] values;
-        final int[] counts;
-
-        int position = 0;
-        long current = Integer.MIN_VALUE;
-        int total;
-
-        public ReaderAggregator(LongFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
-        }
-
-        @Override
-        public void onOrdinal(int docId, int ordinal) {
-            counts[ordinal]++;
-            total++;
-        }
-
-        public boolean nextPosition() {
-            if (++position >= values.length) {
-                return false;
-            }
-            current = values[position];
-            return true;
-        }
-    }
-
-    public static class AggregatorPriorityQueue extends PriorityQueue<ReaderAggregator> {
-
-        public AggregatorPriorityQueue(int size) {
-            super(size);
-        }
-
-        @Override
-        protected boolean lessThan(ReaderAggregator a, ReaderAggregator b) {
-            return a.current < b.current;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java
deleted file mode 100644
index e5cd4ace763..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.shorts;
-
-import com.google.common.collect.ImmutableList;
-import gnu.trove.iterator.TShortIntIterator;
-import gnu.trove.map.hash.TShortIntHashMap;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.text.StringText;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.InternalTermsFacet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class InternalShortTermsFacet extends InternalTermsFacet {
-
-    private static final String STREAM_TYPE = "sTerms";
-
-    public static void registerStream() {
-        Streams.registerStream(STREAM, STREAM_TYPE);
-    }
-
-    static Stream STREAM = new Stream() {
-        @Override
-        public Facet readFacet(String type, StreamInput in) throws IOException {
-            return readTermsFacet(in);
-        }
-    };
-
-    @Override
-    public String streamType() {
-        return STREAM_TYPE;
-    }
-
-    public static class ShortEntry implements Entry {
-
-        short term;
-        int count;
-
-        public ShortEntry(short term, int count) {
-            this.term = term;
-            this.count = count;
-        }
-
-        public Text term() {
-            return new StringText(Short.toString(term));
-        }
-
-        public Text getTerm() {
-            return term();
-        }
-
-        @Override
-        public Number termAsNumber() {
-            return term;
-        }
-
-        @Override
-        public Number getTermAsNumber() {
-            return termAsNumber();
-        }
-
-        public int count() {
-            return count;
-        }
-
-        public int getCount() {
-            return count();
-        }
-
-        @Override
-        public int compareTo(Entry o) {
-            short anotherVal = ((ShortEntry) o).term;
-            int i = term - anotherVal;
-            if (i == 0) {
-                i = count - o.count();
-                if (i == 0) {
-                    i = System.identityHashCode(this) - System.identityHashCode(o);
-                }
-            }
-            return i;
-        }
-    }
-
-    private String name;
-
-    int requiredSize;
-
-    long missing;
-    long total;
-
-    Collection<ShortEntry> entries = ImmutableList.of();
-
-    ComparatorType comparatorType;
-
-    InternalShortTermsFacet() {
-    }
-
-    public InternalShortTermsFacet(String name, ComparatorType comparatorType, int requiredSize, Collection<ShortEntry> entries, long missing, long total) {
-        this.name = name;
-        this.comparatorType = comparatorType;
-        this.requiredSize = requiredSize;
-        this.entries = entries;
-        this.missing = missing;
-        this.total = total;
-    }
-
-    @Override
-    public String name() {
-        return this.name;
-    }
-
-    @Override
-    public String getName() {
-        return this.name;
-    }
-
-    @Override
-    public String type() {
-        return TYPE;
-    }
-
-    @Override
-    public String getType() {
-        return type();
-    }
-
-    @Override
-    public List<ShortEntry> entries() {
-        if (!(entries instanceof List)) {
-            entries = ImmutableList.copyOf(entries);
-        }
-        return (List<ShortEntry>) entries;
-    }
-
-    @Override
-    public List<ShortEntry> getEntries() {
-        return entries();
-    }
-
-    @SuppressWarnings({"unchecked"})
-    @Override
-    public Iterator<Entry> iterator() {
-        return (Iterator) entries.iterator();
-    }
-
-    @Override
-    public long missingCount() {
-        return this.missing;
-    }
-
-    @Override
-    public long getMissingCount() {
-        return missingCount();
-    }
-
-    @Override
-    public long totalCount() {
-        return this.total;
-    }
-
-    @Override
-    public long getTotalCount() {
-        return totalCount();
-    }
-
-    @Override
-    public long otherCount() {
-        long other = total;
-        for (Entry entry : entries) {
-            other -= entry.count();
-        }
-        return other;
-    }
-
-    @Override
-    public long getOtherCount() {
-        return otherCount();
-    }
-
-    @Override
-    public Facet reduce(String name, List<Facet> facets) {
-        if (facets.size() == 1) {
-            return facets.get(0);
-        }
-        InternalShortTermsFacet first = (InternalShortTermsFacet) facets.get(0);
-        TShortIntHashMap aggregated = CacheRecycler.popShortIntMap();
-        long missing = 0;
-        long total = 0;
-        for (Facet facet : facets) {
-            InternalShortTermsFacet mFacet = (InternalShortTermsFacet) facet;
-            missing += mFacet.missingCount();
-            total += mFacet.totalCount();
-            for (ShortEntry entry : mFacet.entries) {
-                aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
-            }
-        }
-
-        BoundedTreeSet<ShortEntry> ordered = new BoundedTreeSet<ShortEntry>(first.comparatorType.comparator(), first.requiredSize);
-        for (TShortIntIterator it = aggregated.iterator(); it.hasNext(); ) {
-            it.advance();
-            ordered.add(new ShortEntry(it.key(), it.value()));
-        }
-        first.entries = ordered;
-        first.missing = missing;
-        first.total = total;
-
-        CacheRecycler.pushShortIntMap(aggregated);
-
-        return first;
-    }
-
-    static final class Fields {
-        static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
-        static final XContentBuilderString MISSING = new XContentBuilderString("missing");
-        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
-        static final XContentBuilderString OTHER = new XContentBuilderString("other");
-        static final XContentBuilderString TERMS = new XContentBuilderString("terms");
-        static final XContentBuilderString TERM = new XContentBuilderString("term");
-        static final XContentBuilderString COUNT = new XContentBuilderString("count");
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(name);
-        builder.field(Fields._TYPE, TermsFacet.TYPE);
-        builder.field(Fields.MISSING, missing);
-        builder.field(Fields.TOTAL, total);
-        builder.field(Fields.OTHER, otherCount());
-        builder.startArray(Fields.TERMS);
-        for (ShortEntry entry : entries) {
-            builder.startObject();
-            builder.field(Fields.TERM, entry.term);
-            builder.field(Fields.COUNT, entry.count());
-            builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-    public static InternalShortTermsFacet readTermsFacet(StreamInput in) throws IOException {
-        InternalShortTermsFacet facet = new InternalShortTermsFacet();
-        facet.readFrom(in);
-        return facet;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        comparatorType = ComparatorType.fromId(in.readByte());
-        requiredSize = in.readVInt();
-        missing = in.readVLong();
-        total = in.readVLong();
-
-        int size = in.readVInt();
-        entries = new ArrayList<ShortEntry>(size);
-        for (int i = 0; i < size; i++) {
-            entries.add(new ShortEntry(in.readShort(), in.readVInt()));
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeByte(comparatorType.id());
-        out.writeVInt(requiredSize);
-        out.writeVLong(missing);
-        out.writeVLong(total);
-
-        out.writeVInt(entries.size());
-        for (ShortEntry entry : entries) {
-            out.writeShort(entry.term);
-            out.writeVInt(entry.count());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java
deleted file mode 100644
index 247729e079c..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortFacetCollector.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.shorts;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.iterator.TShortIntIterator;
-import gnu.trove.map.hash.TShortIntHashMap;
-import gnu.trove.set.hash.TShortHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.shorts.ShortFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Set;
-
-/**
- *
- */
-public class TermsShortFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final FieldDataType fieldDataType;
-
-    private ShortFieldData fieldData;
-
-    private final StaticAggregatorValueProc aggregator;
-
-    private final SearchScript script;
-
-    public TermsShortFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                    ImmutableSet<BytesRef> excluded, String scriptLang, String script, Map<String, Object> params) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms short facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.SHORT) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of short type, can't run terms short facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
-
-        if (this.script == null && excluded.isEmpty()) {
-            aggregator = new StaticAggregatorValueProc(CacheRecycler.popShortIntMap());
-        } else {
-            aggregator = new AggregatorValueProc(CacheRecycler.popShortIntMap(), excluded, this.script);
-        }
-
-        if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    ShortFieldData fieldData = (ShortFieldData) fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
-        }
-    }
-
-    @Override
-    public void setScorer(Scorer scorer) throws IOException {
-        if (script != null) {
-            script.setScorer(scorer);
-        }
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (ShortFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        if (script != null) {
-            script.setNextReader(context);
-        }
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
-    }
-
-    @Override
-    public Facet facet() {
-        TShortIntHashMap facets = aggregator.facets();
-        if (facets.isEmpty()) {
-            CacheRecycler.pushShortIntMap(facets);
-            return new InternalShortTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalShortTermsFacet.ShortEntry>of(), aggregator.missing(), aggregator.total());
-        } else {
-            if (size < EntryPriorityQueue.LIMIT) {
-                EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-                for (TShortIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.insertWithOverflow(new InternalShortTermsFacet.ShortEntry(it.key(), it.value()));
-                }
-                InternalShortTermsFacet.ShortEntry[] list = new InternalShortTermsFacet.ShortEntry[ordered.size()];
-                for (int i = ordered.size() - 1; i >= 0; i--) {
-                    list[i] = (InternalShortTermsFacet.ShortEntry) ordered.pop();
-                }
-                CacheRecycler.pushShortIntMap(facets);
-                return new InternalShortTermsFacet(facetName, comparatorType, size, Arrays.asList(list), aggregator.missing(), aggregator.total());
-            } else {
-                BoundedTreeSet<InternalShortTermsFacet.ShortEntry> ordered = new BoundedTreeSet<InternalShortTermsFacet.ShortEntry>(comparatorType.comparator(), size);
-                for (TShortIntIterator it = facets.iterator(); it.hasNext(); ) {
-                    it.advance();
-                    ordered.add(new InternalShortTermsFacet.ShortEntry(it.key(), it.value()));
-                }
-                CacheRecycler.pushShortIntMap(facets);
-                return new InternalShortTermsFacet(facetName, comparatorType, size, ordered, aggregator.missing(), aggregator.total());
-            }
-        }
-    }
-
-    public static class AggregatorValueProc extends StaticAggregatorValueProc {
-
-        private final SearchScript script;
-
-        private final TShortHashSet excluded;
-
-        public AggregatorValueProc(TShortIntHashMap facets, Set<BytesRef> excluded, SearchScript script) {
-            super(facets);
-            if (excluded == null || excluded.isEmpty()) {
-                this.excluded = null;
-            } else {
-                this.excluded = new TShortHashSet(excluded.size());
-                for (BytesRef s : excluded) {
-                    this.excluded.add(Short.parseShort(s.utf8ToString()));
-                }
-            }
-            this.script = script;
-        }
-
-        @Override
-        public void onValue(int docId, short value) {
-            if (excluded != null && excluded.contains(value)) {
-                return;
-            }
-            if (script != null) {
-                script.setNextDocId(docId);
-                script.setNextVar("term", value);
-                Object scriptValue = script.run();
-                if (scriptValue == null) {
-                    return;
-                }
-                if (scriptValue instanceof Boolean) {
-                    if (!((Boolean) scriptValue)) {
-                        return;
-                    }
-                } else {
-                    value = ((Number) scriptValue).shortValue();
-                }
-            }
-            super.onValue(docId, value);
-        }
-    }
-
-    public static class StaticAggregatorValueProc implements ShortFieldData.ValueInDocProc, ShortFieldData.ValueProc {
-
-        private final TShortIntHashMap facets;
-
-        private int missing;
-
-        private int total;
-
-        public StaticAggregatorValueProc(TShortIntHashMap facets) {
-            this.facets = facets;
-        }
-
-        @Override
-        public void onValue(short value) {
-            facets.putIfAbsent(value, 0);
-        }
-
-        @Override
-        public void onValue(int docId, short value) {
-            facets.adjustOrPutValue(value, 1, 1);
-            total++;
-        }
-
-        @Override
-        public void onMissing(int docId) {
-            missing++;
-        }
-
-        public final TShortIntHashMap facets() {
-            return facets;
-        }
-
-        public final int missing() {
-            return this.missing;
-        }
-
-        public final int total() {
-            return this.total;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortOrdinalsFacetCollector.java
deleted file mode 100644
index 9b0631c3aa1..00000000000
--- a/src/main/java/org/elasticsearch/search/facet/terms/shorts/TermsShortOrdinalsFacetCollector.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.facet.terms.shorts;
-
-import com.google.common.collect.ImmutableSet;
-import gnu.trove.set.hash.TShortHashSet;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.CacheRecycler;
-import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.shorts.ShortFieldData;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.search.facet.AbstractFacetCollector;
-import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- *
- */
-public class TermsShortOrdinalsFacetCollector extends AbstractFacetCollector {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
-
-    private final TermsFacet.ComparatorType comparatorType;
-
-    private final int size;
-
-    private final int numberOfShards;
-
-    private final int minCount;
-
-    private final FieldDataType fieldDataType;
-
-    private ShortFieldData fieldData;
-
-    private final List<ReaderAggregator> aggregators;
-
-    private ReaderAggregator current;
-
-    long missing;
-    long total;
-
-    private final TShortHashSet excluded;
-
-    public TermsShortOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                            ImmutableSet<BytesRef> excluded) {
-        super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
-        this.size = size;
-        this.comparatorType = comparatorType;
-        this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms short facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.SHORT) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of short type, can't run terms short facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
-        if (excluded == null || excluded.isEmpty()) {
-            this.excluded = null;
-        } else {
-            this.excluded = new TShortHashSet(excluded.size());
-            for (BytesRef s : excluded) {
-                this.excluded.add(Short.parseShort(s.utf8ToString()));
-            }
-        }
-
-        // minCount is offset by -1
-        if (allTerms) {
-            minCount = -1;
-        } else {
-            minCount = 0;
-        }
-
-        this.aggregators = new ArrayList<ReaderAggregator>(context.searcher().getIndexReader().leaves().size());
-    }
-
-    @Override
-    protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-        fieldData = (ShortFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
-    }
-
-    @Override
-    protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
-    }
-
-    @Override
-    public Facet facet() {
-        if (current != null) {
-            missing += current.counts[0];
-            total += current.total - current.counts[0];
-            // if we have values for this one, add it
-            if (current.values.length > 1) {
-                aggregators.add(current);
-            }
-        }
-
-        AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
-
-        for (ReaderAggregator aggregator : aggregators) {
-            if (aggregator.nextPosition()) {
-                queue.add(aggregator);
-            }
-        }
-
-        // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
-        if (size < EntryPriorityQueue.LIMIT) {
-            // optimize to use priority size
-            EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-
-            while (queue.size() > 0) {
-                ReaderAggregator agg = queue.top();
-                short value = agg.current;
-                int count = 0;
-                do {
-                    count += agg.counts[agg.position];
-                    if (agg.nextPosition()) {
-                        agg = queue.updateTop();
-                    } else {
-                        // we are done with this reader
-                        queue.pop();
-                        agg = queue.top();
-                    }
-                } while (agg != null && value == agg.current);
-
-                if (count > minCount) {
-                    if (excluded == null || !excluded.contains(value)) {
-                        InternalShortTermsFacet.ShortEntry entry = new InternalShortTermsFacet.ShortEntry(value, count);
-                        ordered.insertWithOverflow(entry);
-                    }
-                }
-            }
-            InternalShortTermsFacet.ShortEntry[] list = new InternalShortTermsFacet.ShortEntry[ordered.size()];
-            for (int i = ordered.size() - 1; i >= 0; i--) {
-                list[i] = (InternalShortTermsFacet.ShortEntry) ordered.pop();
-            }
-
-            for (ReaderAggregator aggregator : aggregators) {
-                CacheRecycler.pushIntArray(aggregator.counts);
-            }
-
-            return new InternalShortTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
-        }
-
-        BoundedTreeSet<InternalShortTermsFacet.ShortEntry> ordered = new BoundedTreeSet<InternalShortTermsFacet.ShortEntry>(comparatorType.comparator(), size);
-
-        while (queue.size() > 0) {
-            ReaderAggregator agg = queue.top();
-            short value = agg.current;
-            int count = 0;
-            do {
-                count += agg.counts[agg.position];
-                if (agg.nextPosition()) {
-                    agg = queue.updateTop();
-                } else {
-                    // we are done with this reader
-                    queue.pop();
-                    agg = queue.top();
-                }
-            } while (agg != null && value == agg.current);
-
-            if (count > minCount) {
-                if (excluded == null || !excluded.contains(value)) {
-                    InternalShortTermsFacet.ShortEntry entry = new InternalShortTermsFacet.ShortEntry(value, count);
-                    ordered.add(entry);
-                }
-            }
-        }
-
-        for (ReaderAggregator aggregator : aggregators) {
-            CacheRecycler.pushIntArray(aggregator.counts);
-        }
-
-        return new InternalShortTermsFacet(facetName, comparatorType, size, ordered, missing, total);
-    }
-
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
-
-        final short[] values;
-        final int[] counts;
-
-        int position = 0;
-        short current;
-        int total;
-
-        public ReaderAggregator(ShortFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
-        }
-
-        @Override
-        public void onOrdinal(int docId, int ordinal) {
-            counts[ordinal]++;
-            total++;
-        }
-
-        public boolean nextPosition() {
-            if (++position >= values.length) {
-                return false;
-            }
-            current = values[position];
-            return true;
-        }
-    }
-
-    public static class AggregatorPriorityQueue extends PriorityQueue<ReaderAggregator> {
-
-        public AggregatorPriorityQueue(int size) {
-            super(size);
-        }
-
-        @Override
-        protected boolean lessThan(ReaderAggregator a, ReaderAggregator b) {
-            return a.current < b.current;
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java
index 013665e3b39..641914745b1 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetCollector.java
@@ -28,10 +28,10 @@ import org.apache.lucene.search.Scorer;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.HashedBytesValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
@@ -41,7 +41,6 @@ import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -50,72 +49,59 @@ import java.util.regex.Pattern;
  */
 public class FieldsTermsStringFacetCollector extends AbstractFacetCollector {
 
-    private final FieldDataCache fieldDataCache;
-
-    private final String[] indexFieldsNames;
-
     private final InternalStringTermsFacet.ComparatorType comparatorType;
 
     private final int size;
 
     private final int numberOfShards;
 
-    private final FieldDataType[] fieldsDataType;
+    private final IndexFieldData[] indexFieldDatas;
+    private HashedBytesValues[] values;
 
-    private FieldData[] fieldsData;
-
-    private final StaticAggregatorValueProc aggregator;
+    private final StaticAggregatorValueProc[] aggregators;
 
     private final SearchScript script;
 
     public FieldsTermsStringFacetCollector(String facetName, String[] fieldsNames, int size, InternalStringTermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                           ImmutableSet<BytesRef> excluded, Pattern pattern, String scriptLang, String script, Map<String, Object> params) {
+                                           ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
+        this.script = script;
 
-        fieldsDataType = new FieldDataType[fieldsNames.length];
-        fieldsData = new FieldData[fieldsNames.length];
-        indexFieldsNames = new String[fieldsNames.length];
+        indexFieldDatas = new IndexFieldData[fieldsNames.length];
+        values = new HashedBytesValues[fieldsNames.length];
+        aggregators = new StaticAggregatorValueProc[fieldsNames.length];
 
+        TObjectIntHashMap<HashedBytesRef> map = CacheRecycler.popObjectIntMap();
         for (int i = 0; i < fieldsNames.length; i++) {
-            MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldsNames[i]);
-            if (smartMappers == null || !smartMappers.hasMapper()) {
-                this.indexFieldsNames[i] = fieldsNames[i];
-                this.fieldsDataType[i] = FieldDataType.DefaultTypes.STRING;
+            FieldMapper mapper = context.smartNameFieldMapper(fieldsNames[i]);
+            if (mapper == null) {
+                throw new FacetPhaseExecutionException(facetName, "failed to find mapping for [" + fieldsNames[i] + "]");
+            }
+            indexFieldDatas[i] = context.fieldData().getForField(mapper);
+            if (excluded.isEmpty() && pattern == null && this.script == null) {
+                aggregators[i] = new StaticAggregatorValueProc(map);
             } else {
-                this.indexFieldsNames[i] = smartMappers.mapper().names().indexName();
-                this.fieldsDataType[i] = smartMappers.mapper().fieldDataType();
-            }
-
-        }
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
-
-        if (excluded.isEmpty() && pattern == null && this.script == null) {
-            aggregator = new StaticAggregatorValueProc(CacheRecycler.<BytesRef>popObjectIntMap());
-        } else {
-            aggregator = new AggregatorValueProc(CacheRecycler.<BytesRef>popObjectIntMap(), excluded, pattern, this.script);
-        }
-
-        if (allTerms) {
-            try {
-                for (int i = 0; i < fieldsNames.length; i++) {
-                    for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                        FieldData fieldData = fieldDataCache.cache(fieldsDataType[i], readerContext.reader(), indexFieldsNames[i]);
-                        fieldData.forEachValue(aggregator);
-                    }
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
+                aggregators[i] = new AggregatorValueProc(map, excluded, pattern, this.script);
             }
         }
+
+
+        // TODO: we need to support this flag with the new field data...
+//        if (allTerms) {
+//            try {
+//                for (int i = 0; i < fieldsNames.length; i++) {
+//                    for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
+//                        FieldData fieldData = fieldDataCache.cache(fieldsDataType[i], readerContext.reader(), indexFieldsNames[i]);
+//                        fieldData.forEachValue(aggregator);
+//                    }
+//                }
+//            } catch (Exception e) {
+//                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
+//            }
+//        }
     }
 
     @Override
@@ -127,8 +113,9 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        for (int i = 0; i < indexFieldsNames.length; i++) {
-            fieldsData[i] = fieldDataCache.cache(fieldsDataType[i], context.reader(), indexFieldsNames[i]);
+        for (int i = 0; i < indexFieldDatas.length; i++) {
+            values[i] = indexFieldDatas[i].load(context).getHashedBytesValues();
+            aggregators[i].values = values[i];
         }
         if (script != null) {
             script.setNextReader(context);
@@ -137,38 +124,44 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        for (FieldData fieldData : fieldsData) {
-            fieldData.forEachValueInDoc(doc, aggregator);
+        for (int i = 0; i < values.length; i++) {
+            values[i].forEachValueInDoc(doc, aggregators[i]);
         }
     }
 
     @Override
     public Facet facet() {
-        TObjectIntHashMap<BytesRef> facets = aggregator.facets();
+        TObjectIntHashMap<HashedBytesRef> facets = aggregators[0].facets(); // we share the map between all aggregators
+        long totalMissing = 0;
+        long total = 0;
+        for (StaticAggregatorValueProc aggregator : aggregators) {
+            totalMissing += aggregator.missing();
+            total += aggregator.total();
+        }
         if (facets.isEmpty()) {
             CacheRecycler.pushObjectIntMap(facets);
-            return new InternalStringTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalStringTermsFacet.TermEntry>of(), aggregator.missing(), aggregator.total());
+            return new InternalStringTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalStringTermsFacet.TermEntry>of(), totalMissing, total);
         } else {
             if (size < EntryPriorityQueue.LIMIT) {
                 EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-                for (TObjectIntIterator<BytesRef> it = facets.iterator(); it.hasNext(); ) {
+                for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
                     it.advance();
-                    ordered.insertWithOverflow(new InternalStringTermsFacet.TermEntry(it.key(), it.value()));
+                    ordered.insertWithOverflow(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
                 }
                 InternalStringTermsFacet.TermEntry[] list = new InternalStringTermsFacet.TermEntry[ordered.size()];
                 for (int i = ordered.size() - 1; i >= 0; i--) {
                     list[i] = ((InternalStringTermsFacet.TermEntry) ordered.pop());
                 }
                 CacheRecycler.pushObjectIntMap(facets);
-                return new InternalStringTermsFacet(facetName, comparatorType, size, Arrays.asList(list), aggregator.missing(), aggregator.total());
+                return new InternalStringTermsFacet(facetName, comparatorType, size, Arrays.asList(list), totalMissing, total);
             } else {
                 BoundedTreeSet<InternalStringTermsFacet.TermEntry> ordered = new BoundedTreeSet<InternalStringTermsFacet.TermEntry>(comparatorType.comparator(), size);
-                for (TObjectIntIterator<BytesRef> it = facets.iterator(); it.hasNext(); ) {
+                for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
                     it.advance();
-                    ordered.add(new InternalStringTermsFacet.TermEntry(it.key(), it.value()));
+                    ordered.add(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
                 }
                 CacheRecycler.pushObjectIntMap(facets);
-                return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, aggregator.missing(), aggregator.total());
+                return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, totalMissing, total);
             }
         }
     }
@@ -181,7 +174,7 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector {
 
         private final SearchScript script;
 
-        public AggregatorValueProc(TObjectIntHashMap<BytesRef> facets, ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
+        public AggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets, ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
             super(facets);
             this.excluded = excluded;
             this.matcher = pattern != null ? pattern.matcher("") : null;
@@ -189,19 +182,19 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector {
         }
 
         @Override
-        public void onValue(int docId, BytesRef value) {
-            if (excluded != null && excluded.contains(value)) {
+        public void onValue(int docId, HashedBytesRef value) {
+            if (excluded != null && excluded.contains(value.bytes)) {
                 return;
             }
 
             // LUCENE 4 UPGRADE: use Lucene's RegexCapabilities
-            if (matcher != null && !matcher.reset(value.utf8ToString()).matches()) {
+            if (matcher != null && !matcher.reset(value.bytes.utf8ToString()).matches()) {
                 return;
             }
             if (script != null) {
                 script.setNextDocId(docId);
                 // LUCENE 4 UPGRADE: needs optimization
-                script.setNextVar("term", value.utf8ToString());
+                script.setNextVar("term", value.bytes.utf8ToString());
                 Object scriptValue = script.run();
                 if (scriptValue == null) {
                     return;
@@ -212,33 +205,29 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector {
                     }
                 } else {
                     // LUCENE 4 UPGRADE: make script return BR?
-                    value = new BytesRef(scriptValue.toString());
+                    value = new HashedBytesRef(scriptValue.toString());
                 }
             }
             super.onValue(docId, value);
         }
     }
 
-    public static class StaticAggregatorValueProc implements FieldData.StringValueInDocProc, FieldData.StringValueProc {
+    public static class StaticAggregatorValueProc implements HashedBytesValues.ValueInDocProc {
 
         // LUCENE 4 UPGRADE: check if hashcode is not too expensive
-        private final TObjectIntHashMap<BytesRef> facets;
+        private final TObjectIntHashMap<HashedBytesRef> facets;
+        HashedBytesValues values;
 
         private int missing;
         private int total;
 
-        public StaticAggregatorValueProc(TObjectIntHashMap<BytesRef> facets) {
+        public StaticAggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets) {
             this.facets = facets;
         }
 
         @Override
-        public void onValue(BytesRef value) {
-            facets.putIfAbsent(value, 0);
-        }
-
-        @Override
-        public void onValue(int docId, BytesRef value) {
-            facets.adjustOrPutValue(value, 1, 1);
+        public void onValue(int docId, HashedBytesRef value) {
+            facets.adjustOrPutValue(values.makeSafe(value), 1, 1);
             total++;
         }
 
@@ -247,7 +236,7 @@ public class FieldsTermsStringFacetCollector extends AbstractFacetCollector {
             missing++;
         }
 
-        public final TObjectIntHashMap<BytesRef> facets() {
+        public final TObjectIntHashMap<HashedBytesRef> facets() {
             return facets;
         }
 
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java
index 93cdc7a3ad0..7ca6c127eba 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetCollector.java
@@ -28,24 +28,18 @@ import org.apache.lucene.search.Scorer;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.HashedBytesValues;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
-import org.elasticsearch.search.facet.FacetPhaseExecutionException;
 import org.elasticsearch.search.facet.terms.TermsFacet;
 import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.ArrayDeque;
 import java.util.Arrays;
-import java.util.Deque;
-import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -54,17 +48,7 @@ import java.util.regex.Pattern;
  */
 public class TermsStringFacetCollector extends AbstractFacetCollector {
 
-    static ThreadLocal<ThreadLocals.CleanableValue<Deque<TObjectIntHashMap<String>>>> cache = new ThreadLocal<ThreadLocals.CleanableValue<Deque<TObjectIntHashMap<String>>>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<Deque<TObjectIntHashMap<String>>> initialValue() {
-            return new ThreadLocals.CleanableValue<Deque<TObjectIntHashMap<java.lang.String>>>(new ArrayDeque<TObjectIntHashMap<String>>());
-        }
-    };
-
-
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
+    private final IndexFieldData indexFieldData;
 
     private final TermsFacet.ComparatorType comparatorType;
 
@@ -72,57 +56,37 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
 
     private final int numberOfShards;
 
-    private final FieldDataType fieldDataType;
-
-    private FieldData fieldData;
+    private HashedBytesValues values;
 
     private final StaticAggregatorValueProc aggregator;
 
     private final SearchScript script;
 
-    public TermsStringFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
-                                     ImmutableSet<BytesRef> excluded, Pattern pattern, String scriptLang, String script, Map<String, Object> params) {
+    public TermsStringFacetCollector(String facetName, IndexFieldData indexFieldData, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
+                                     ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
+        this.indexFieldData = indexFieldData;
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
-
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            this.indexFieldName = fieldName;
-            this.fieldDataType = FieldDataType.DefaultTypes.STRING;
-        } else {
-            // add type filter if there is exact doc mapper associated with it
-            if (smartMappers.hasDocMapper()) {
-                setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-            }
-
-            this.indexFieldName = smartMappers.mapper().names().indexName();
-            this.fieldDataType = smartMappers.mapper().fieldDataType();
-        }
-
-        if (script != null) {
-            this.script = context.scriptService().search(context.lookup(), scriptLang, script, params);
-        } else {
-            this.script = null;
-        }
+        this.script = script;
 
         if (excluded.isEmpty() && pattern == null && this.script == null) {
-            aggregator = new StaticAggregatorValueProc(CacheRecycler.<BytesRef>popObjectIntMap());
+            aggregator = new StaticAggregatorValueProc(CacheRecycler.<HashedBytesRef>popObjectIntMap());
         } else {
-            aggregator = new AggregatorValueProc(CacheRecycler.<BytesRef>popObjectIntMap(), excluded, pattern, this.script);
+            aggregator = new AggregatorValueProc(CacheRecycler.<HashedBytesRef>popObjectIntMap(), excluded, pattern, this.script);
         }
 
         if (allTerms) {
-            try {
-                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
-                    FieldData fieldData = fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
-                    fieldData.forEachValue(aggregator);
-                }
-            } catch (Exception e) {
-                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
-            }
+            // TODO: we need to support this back with the new field data!
+//            try {
+//                for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
+//                    FieldData fieldData = fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
+//                    fieldData.forEachValue(aggregator);
+//                }
+//            } catch (Exception e) {
+//                throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
+//            }
         }
     }
 
@@ -135,7 +99,8 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doSetNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
+        values = indexFieldData.load(context).getHashedBytesValues();
+        aggregator.values = values;
         if (script != null) {
             script.setNextReader(context);
         }
@@ -143,21 +108,21 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachValueInDoc(doc, aggregator);
+        values.forEachValueInDoc(doc, aggregator);
     }
 
     @Override
     public Facet facet() {
-        TObjectIntHashMap<BytesRef> facets = aggregator.facets();
+        TObjectIntHashMap<HashedBytesRef> facets = aggregator.facets();
         if (facets.isEmpty()) {
             CacheRecycler.pushObjectIntMap(facets);
             return new InternalStringTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalStringTermsFacet.TermEntry>of(), aggregator.missing(), aggregator.total());
         } else {
             if (size < EntryPriorityQueue.LIMIT) {
                 EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
-                for (TObjectIntIterator<BytesRef> it = facets.iterator(); it.hasNext(); ) {
+                for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
                     it.advance();
-                    ordered.insertWithOverflow(new InternalStringTermsFacet.TermEntry(it.key(), it.value()));
+                    ordered.insertWithOverflow(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
                 }
                 InternalStringTermsFacet.TermEntry[] list = new InternalStringTermsFacet.TermEntry[ordered.size()];
                 for (int i = ordered.size() - 1; i >= 0; i--) {
@@ -167,9 +132,9 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
                 return new InternalStringTermsFacet(facetName, comparatorType, size, Arrays.asList(list), aggregator.missing(), aggregator.total());
             } else {
                 BoundedTreeSet<InternalStringTermsFacet.TermEntry> ordered = new BoundedTreeSet<InternalStringTermsFacet.TermEntry>(comparatorType.comparator(), size);
-                for (TObjectIntIterator<BytesRef> it = facets.iterator(); it.hasNext(); ) {
+                for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
                     it.advance();
-                    ordered.add(new InternalStringTermsFacet.TermEntry(it.key(), it.value()));
+                    ordered.add(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
                 }
                 CacheRecycler.pushObjectIntMap(facets);
                 return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, aggregator.missing(), aggregator.total());
@@ -185,7 +150,7 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
 
         private final SearchScript script;
 
-        public AggregatorValueProc(TObjectIntHashMap<BytesRef> facets, ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
+        public AggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets, ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
             super(facets);
             this.excluded = excluded;
             this.matcher = pattern != null ? pattern.matcher("") : null;
@@ -193,18 +158,18 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
         }
 
         @Override
-        public void onValue(int docId, BytesRef value) {
-            if (excluded != null && excluded.contains(value)) {
+        public void onValue(int docId, HashedBytesRef value) {
+            if (excluded != null && excluded.contains(value.bytes)) {
                 return;
             }
             // LUCENE 4 UPGRADE: use Lucene's RegexCapabilities
-            if (matcher != null && !matcher.reset(value.utf8ToString()).matches()) {
+            if (matcher != null && !matcher.reset(value.bytes.utf8ToString()).matches()) {
                 return;
             }
             if (script != null) {
                 script.setNextDocId(docId);
                 // LUCENE 4 UPGRADE: needs optimization
-                script.setNextVar("term", value.utf8ToString());
+                script.setNextVar("term", value.bytes.utf8ToString());
                 Object scriptValue = script.run();
                 if (scriptValue == null) {
                     return;
@@ -215,33 +180,30 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
                     }
                 } else {
                     // LUCENE 4 UPGRADE: should be possible to convert directly to BR
-                    value = new BytesRef(scriptValue.toString());
+                    value = new HashedBytesRef(scriptValue.toString());
                 }
             }
             super.onValue(docId, value);
         }
     }
 
-    public static class StaticAggregatorValueProc implements FieldData.StringValueInDocProc, FieldData.StringValueProc {
+    public static class StaticAggregatorValueProc implements HashedBytesValues.ValueInDocProc {
 
         // LUCENE 4 UPGRADE: check if hashcode is not too expensive
-        private final TObjectIntHashMap<BytesRef> facets;
+        private final TObjectIntHashMap<HashedBytesRef> facets;
 
+        HashedBytesValues values;
         private int missing = 0;
         private int total = 0;
 
-        public StaticAggregatorValueProc(TObjectIntHashMap<BytesRef> facets) {
+        public StaticAggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets) {
             this.facets = facets;
         }
 
         @Override
-        public void onValue(BytesRef value) {
-            facets.putIfAbsent(value, 0);
-        }
-
-        @Override
-        public void onValue(int docId, BytesRef value) {
-            facets.adjustOrPutValue(value, 1, 1);
+        public void onValue(int docId, HashedBytesRef value) {
+            // we have to "makeSafe", even if it exists, since it might not..., need to find a way to optimize it
+            facets.adjustOrPutValue(values.makeSafe(value), 1, 1);
             total++;
         }
 
@@ -250,7 +212,7 @@ public class TermsStringFacetCollector extends AbstractFacetCollector {
             missing++;
         }
 
-        public final TObjectIntHashMap<BytesRef> facets() {
+        public final TObjectIntHashMap<HashedBytesRef> facets() {
             return facets;
         }
 
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetCollector.java b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetCollector.java
index 42856f356ea..603fc109608 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetCollector.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetCollector.java
@@ -23,14 +23,11 @@ import com.google.common.collect.ImmutableSet;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.CacheRecycler;
 import org.elasticsearch.common.collect.BoundedTreeSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.strings.StringFieldData;
-import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
+import org.elasticsearch.index.fielddata.OrdinalsBytesValues;
+import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.search.facet.AbstractFacetCollector;
 import org.elasticsearch.search.facet.Facet;
 import org.elasticsearch.search.facet.terms.TermsFacet;
@@ -49,9 +46,7 @@ import java.util.regex.Pattern;
  */
 public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
 
-    private final FieldDataCache fieldDataCache;
-
-    private final String indexFieldName;
+    private final IndexOrdinalFieldData indexFieldData;
 
     private final TermsFacet.ComparatorType comparatorType;
 
@@ -61,9 +56,7 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
 
     private final int minCount;
 
-    private final FieldDataType fieldDataType;
-
-    private StringFieldData fieldData;
+    private OrdinalsBytesValues values;
 
     private final List<ReaderAggregator> aggregators;
 
@@ -76,30 +69,14 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
 
     private final Matcher matcher;
 
-    public TermsStringOrdinalsFacetCollector(String facetName, String fieldName, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
+    public TermsStringOrdinalsFacetCollector(String facetName, IndexOrdinalFieldData indexFieldData, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
                                              ImmutableSet<BytesRef> excluded, Pattern pattern) {
         super(facetName);
-        this.fieldDataCache = context.fieldDataCache();
+        this.indexFieldData = indexFieldData;
         this.size = size;
         this.comparatorType = comparatorType;
         this.numberOfShards = context.numberOfShards();
 
-        MapperService.SmartNameFieldMappers smartMappers = context.smartFieldMappers(fieldName);
-        if (smartMappers == null || !smartMappers.hasMapper()) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] doesn't have a type, can't run terms long facet collector on it");
-        }
-        // add type filter if there is exact doc mapper associated with it
-        if (smartMappers.explicitTypeInNameWithDocMapper()) {
-            setFilter(context.filterCache().cache(smartMappers.docMapper().typeFilter()));
-        }
-
-        if (smartMappers.mapper().fieldDataType() != FieldDataType.DefaultTypes.STRING) {
-            throw new ElasticSearchIllegalArgumentException("Field [" + fieldName + "] is not of string type, can't run terms string facet collector on it");
-        }
-
-        this.indexFieldName = smartMappers.mapper().names().indexName();
-        this.fieldDataType = smartMappers.mapper().fieldDataType();
-
         if (excluded == null || excluded.isEmpty()) {
             this.excluded = null;
         } else {
@@ -122,17 +99,17 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
         if (current != null) {
             missing += current.counts[0];
             total += current.total - current.counts[0];
-            if (current.values.length > 1) {
+            if (current.values.ordinals().getNumOrds() > 1) {
                 aggregators.add(current);
             }
         }
-        fieldData = (StringFieldData) fieldDataCache.cache(fieldDataType, context.reader(), indexFieldName);
-        current = new ReaderAggregator(fieldData);
+        values = indexFieldData.load(context).getBytesValues();
+        current = new ReaderAggregator(values);
     }
 
     @Override
     protected void doCollect(int doc) throws IOException {
-        fieldData.forEachOrdinalInDoc(doc, current);
+        values.ordinals().forEachOrdinalInDoc(doc, current);
     }
 
     @Override
@@ -141,7 +118,7 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
             missing += current.counts[0];
             total += current.total - current.counts[0];
             // if we have values for this one, add it
-            if (current.values.length > 1) {
+            if (current.values.ordinals().getNumOrds() > 1) {
                 aggregators.add(current);
             }
         }
@@ -161,7 +138,7 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
 
             while (queue.size() > 0) {
                 ReaderAggregator agg = queue.top();
-                BytesRef value = agg.current;
+                BytesRef value = agg.values.makeSafe(agg.current); // we need to makeSafe it, since we end up pushing it... (can we get around this?)
                 int count = 0;
                 do {
                     count += agg.counts[agg.position];
@@ -202,7 +179,7 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
 
         while (queue.size() > 0) {
             ReaderAggregator agg = queue.top();
-            BytesRef value = agg.current;
+            BytesRef value = agg.values.makeSafe(agg.current); // we need to makeSafe it, since we end up pushing it... (can we work around that?)
             int count = 0;
             do {
                 count += agg.counts[agg.position];
@@ -236,18 +213,18 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
         return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, missing, total);
     }
 
-    public static class ReaderAggregator implements FieldData.OrdinalInDocProc {
+    public static class ReaderAggregator implements Ordinals.Docs.OrdinalInDocProc {
 
-        final BytesRef[] values;
+        final OrdinalsBytesValues values;
         final int[] counts;
 
         int position = 0;
         BytesRef current;
         int total;
 
-        public ReaderAggregator(StringFieldData fieldData) {
-            this.values = fieldData.values();
-            this.counts = CacheRecycler.popIntArray(fieldData.values().length);
+        public ReaderAggregator(OrdinalsBytesValues values) {
+            this.values = values;
+            this.counts = CacheRecycler.popIntArray(values.ordinals().getNumOrds());
         }
 
         @Override
@@ -257,10 +234,10 @@ public class TermsStringOrdinalsFacetCollector extends AbstractFacetCollector {
         }
 
         public boolean nextPosition() {
-            if (++position >= values.length) {
+            if (++position >= values.ordinals().getNumOrds()) {
                 return false;
             }
-            current = values[position];
+            current = values.getValueByOrd(position);
             return true;
         }
     }
diff --git a/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java b/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java
index c1456d84ed8..ae593419299 100644
--- a/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java
+++ b/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java
@@ -34,11 +34,8 @@ import org.elasticsearch.search.facet.query.QueryFacet;
 import org.elasticsearch.search.facet.range.RangeFacet;
 import org.elasticsearch.search.facet.statistical.StatisticalFacet;
 import org.elasticsearch.search.facet.terms.TermsFacet;
-import org.elasticsearch.search.facet.terms.bytes.InternalByteTermsFacet;
 import org.elasticsearch.search.facet.terms.doubles.InternalDoubleTermsFacet;
-import org.elasticsearch.search.facet.terms.ints.InternalIntTermsFacet;
 import org.elasticsearch.search.facet.terms.longs.InternalLongTermsFacet;
-import org.elasticsearch.search.facet.terms.shorts.InternalShortTermsFacet;
 import org.elasticsearch.search.facet.termsstats.TermsStatsFacet;
 import org.elasticsearch.test.integration.AbstractNodesTests;
 import org.joda.time.DateTimeZone;
@@ -535,7 +532,6 @@ public class SimpleFacetsTests extends AbstractNodesTests {
                     .execute().actionGet();
 
             facet = searchResponse.facets().facet("facet1");
-            assertThat(facet, instanceOf(InternalByteTermsFacet.class));
             assertThat(facet.name(), equalTo("facet1"));
             assertThat(facet.entries().size(), equalTo(1));
             assertThat(facet.entries().get(0).term().string(), equalTo("111"));
@@ -547,7 +543,6 @@ public class SimpleFacetsTests extends AbstractNodesTests {
                     .execute().actionGet();
 
             facet = searchResponse.facets().facet("facet1");
-            assertThat(facet, instanceOf(InternalIntTermsFacet.class));
             assertThat(facet.name(), equalTo("facet1"));
             assertThat(facet.entries().size(), equalTo(1));
             assertThat(facet.entries().get(0).term().string(), equalTo("111"));
@@ -559,7 +554,6 @@ public class SimpleFacetsTests extends AbstractNodesTests {
                     .execute().actionGet();
 
             facet = searchResponse.facets().facet("facet1");
-            assertThat(facet, instanceOf(InternalShortTermsFacet.class));
             assertThat(facet.name(), equalTo("facet1"));
             assertThat(facet.entries().size(), equalTo(1));
             assertThat(facet.entries().get(0).term().string(), equalTo("111"));
@@ -729,20 +723,21 @@ public class SimpleFacetsTests extends AbstractNodesTests {
             assertThat(facet.entries().get(3).term().string(), anyOf(equalTo("zzz"), equalTo("xxx")));
             assertThat(facet.entries().get(3).count(), equalTo(1));
 
-            searchResponse = client.prepareSearch()
-                    .setQuery(termQuery("xxx", "yyy")) // don't match anything
-                    .addFacet(termsFacet("facet1").field("tag").size(10).allTerms(true).executionHint(executionHint))
-                    .execute().actionGet();
-
-            facet = searchResponse.facets().facet("facet1");
-            assertThat(facet.name(), equalTo("facet1"));
-            assertThat(facet.entries().size(), equalTo(3));
-            assertThat(facet.entries().get(0).term().string(), anyOf(equalTo("xxx"), equalTo("yyy"), equalTo("zzz")));
-            assertThat(facet.entries().get(0).count(), equalTo(0));
-            assertThat(facet.entries().get(1).term().string(), anyOf(equalTo("xxx"), equalTo("yyy"), equalTo("zzz")));
-            assertThat(facet.entries().get(1).count(), equalTo(0));
-            assertThat(facet.entries().get(2).term().string(), anyOf(equalTo("xxx"), equalTo("yyy"), equalTo("zzz")));
-            assertThat(facet.entries().get(2).count(), equalTo(0));
+            // TODO: support allTerms with the new field data
+//            searchResponse = client.prepareSearch()
+//                    .setQuery(termQuery("xxx", "yyy")) // don't match anything
+//                    .addFacet(termsFacet("facet1").field("tag").size(10).allTerms(true).executionHint(executionHint))
+//                    .execute().actionGet();
+//
+//            facet = searchResponse.facets().facet("facet1");
+//            assertThat(facet.name(), equalTo("facet1"));
+//            assertThat(facet.entries().size(), equalTo(3));
+//            assertThat(facet.entries().get(0).term().string(), anyOf(equalTo("xxx"), equalTo("yyy"), equalTo("zzz")));
+//            assertThat(facet.entries().get(0).count(), equalTo(0));
+//            assertThat(facet.entries().get(1).term().string(), anyOf(equalTo("xxx"), equalTo("yyy"), equalTo("zzz")));
+//            assertThat(facet.entries().get(1).count(), equalTo(0));
+//            assertThat(facet.entries().get(2).term().string(), anyOf(equalTo("xxx"), equalTo("yyy"), equalTo("zzz")));
+//            assertThat(facet.entries().get(2).count(), equalTo(0));
 
             // Script Field
 

From be1e5becbbf7aa6efa2a30c1b1e5e3633ba0a11e Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 13:45:00 +0100
Subject: [PATCH 28/35] move scripts to use new field data

---
 .../index/fielddata/ScriptDocValues.java      | 84 +++++++++++++++++++
 .../plain/ByteArrayAtomicFieldData.java       |  6 +-
 .../plain/FloatArrayAtomicFieldData.java      | 11 ++-
 .../plain/IntArrayAtomicFieldData.java        |  4 +-
 .../plain/LongArrayAtomicFieldData.java       |  6 +-
 .../plain/ShortArrayAtomicFieldData.java      |  6 +-
 .../index/get/ShardGetService.java            | 16 ++--
 .../index/query/IndexQueryParserService.java  |  6 +-
 .../index/query/QueryParseContext.java        |  8 +-
 .../elasticsearch/script/ScriptService.java   |  6 +-
 .../search/internal/SearchContext.java        |  2 +-
 .../search/lookup/DocLookup.java              | 44 ++++------
 .../search/lookup/SearchLookup.java           |  6 +-
 13 files changed, 145 insertions(+), 60 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index cdf334ce8d1..bd187aa5e94 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -20,8 +20,11 @@
 package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.fielddata.util.*;
 import org.elasticsearch.index.mapper.geo.GeoPoint;
+import org.elasticsearch.index.search.geo.GeoDistance;
+import org.joda.time.MutableDateTime;
 
 /**
  * Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
@@ -176,6 +179,7 @@ public interface ScriptDocValues {
     static class NumericLong implements ScriptDocValues {
 
         private final LongValues values;
+        private final MutableDateTime date = new MutableDateTime(0);
         private int docId;
 
         public NumericLong(LongValues values) {
@@ -196,6 +200,11 @@ public interface ScriptDocValues {
             return values.getValue(docId);
         }
 
+        public MutableDateTime getDate() {
+            date.setMillis(getValue());
+            return date;
+        }
+
         public LongArrayRef getValues() {
             return values.getValues(docId);
         }
@@ -283,5 +292,80 @@ public interface ScriptDocValues {
         public GeoPointArrayRef getValues() {
             return values.getValues(docId);
         }
+
+        public double factorDistance(double lat, double lon) {
+            GeoPoint point = getValue();
+            return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
+        }
+
+        public double factorDistanceWithDefault(double lat, double lon, double defaultValue) {
+            if (isEmpty()) {
+                return defaultValue;
+            }
+            GeoPoint point = getValue();
+            return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
+        }
+
+        public double factorDistance02(double lat, double lon) {
+            GeoPoint point = getValue();
+            return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES) + 1;
+        }
+
+        public double factorDistance13(double lat, double lon) {
+            GeoPoint point = getValue();
+            return GeoDistance.FACTOR.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES) + 2;
+        }
+
+        public double arcDistance(double lat, double lon) {
+            GeoPoint point = getValue();
+            return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
+        }
+
+        public double arcDistanceWithDefault(double lat, double lon, double defaultValue) {
+            if (isEmpty()) {
+                return defaultValue;
+            }
+            GeoPoint point = getValue();
+            return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
+        }
+
+        public double arcDistanceInKm(double lat, double lon) {
+            GeoPoint point = getValue();
+            return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
+        }
+
+        public double arcDistanceInKmWithDefault(double lat, double lon, double defaultValue) {
+            if (isEmpty()) {
+                return defaultValue;
+            }
+            GeoPoint point = getValue();
+            return GeoDistance.ARC.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
+        }
+
+        public double distance(double lat, double lon) {
+            GeoPoint point = getValue();
+            return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
+        }
+
+        public double distanceWithDefault(double lat, double lon, double defaultValue) {
+            if (isEmpty()) {
+                return defaultValue;
+            }
+            GeoPoint point = getValue();
+            return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.MILES);
+        }
+
+        public double distanceInKm(double lat, double lon) {
+            GeoPoint point = getValue();
+            return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
+        }
+
+        public double distanceInKmWithDefault(double lat, double lon, double defaultValue) {
+            if (isEmpty()) {
+                return defaultValue;
+            }
+            GeoPoint point = getValue();
+            return GeoDistance.PLANE.calculate(point.lat(), point.lon(), lat, lon, DistanceUnit.KILOMETERS);
+        }
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
index 3378424be14..a5ef7ee1896 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
@@ -91,7 +91,7 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericByte(getByteValues());
         }
 
         @Override
@@ -458,7 +458,7 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericByte(getByteValues());
         }
 
         @Override
@@ -733,7 +733,7 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericByte(getByteValues());
         }
 
         @Override
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
index afba43c9f3a..9ae133b7861 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
@@ -23,7 +23,10 @@ import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.RamUsage;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
-import org.elasticsearch.index.fielddata.util.*;
+import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
+import org.elasticsearch.index.fielddata.util.FloatArrayRef;
+import org.elasticsearch.index.fielddata.util.IntArrayRef;
+import org.elasticsearch.index.fielddata.util.LongArrayRef;
 
 /**
  */
@@ -88,7 +91,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericFloat(getFloatValues());
         }
 
         @Override
@@ -455,7 +458,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericFloat(getFloatValues());
         }
 
         @Override
@@ -730,7 +733,7 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericFloat(getFloatValues());
         }
 
         @Override
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
index 7c01d8a3df1..0bf31844a40 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
@@ -457,7 +457,7 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericInteger(getIntValues());
         }
 
         @Override
@@ -731,7 +731,7 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericInteger(getIntValues());
         }
 
         @Override
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
index dbaa9f90505..4198bdde0a7 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
@@ -91,7 +91,7 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericLong(getLongValues());
         }
 
         @Override
@@ -452,7 +452,7 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericLong(getLongValues());
         }
 
         @Override
@@ -723,7 +723,7 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericLong(getLongValues());
         }
 
         @Override
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
index 236bb4d89b0..9733326e028 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
@@ -91,7 +91,7 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericShort(getShortValues());
         }
 
         @Override
@@ -458,7 +458,7 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericShort(getShortValues());
         }
 
         @Override
@@ -733,7 +733,7 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
 
         @Override
         public ScriptDocValues getScriptValues() {
-            return new ScriptDocValues.NumericDouble(getDoubleValues());
+            return new ScriptDocValues.NumericShort(getShortValues());
         }
 
         @Override
diff --git a/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/src/main/java/org/elasticsearch/index/get/ShardGetService.java
index 2c81f2519a7..143200d0dcf 100644
--- a/src/main/java/org/elasticsearch/index/get/ShardGetService.java
+++ b/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -28,8 +28,8 @@ import org.elasticsearch.common.lucene.uid.UidField;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
 import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor;
@@ -62,7 +62,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
 
     private final MapperService mapperService;
 
-    private final IndexCache indexCache;
+    private final IndexFieldDataService fieldDataService;
 
     private IndexShard indexShard;
 
@@ -72,11 +72,11 @@ public class ShardGetService extends AbstractIndexShardComponent {
 
     @Inject
     public ShardGetService(ShardId shardId, @IndexSettings Settings indexSettings, ScriptService scriptService,
-                           MapperService mapperService, IndexCache indexCache) {
+                           MapperService mapperService, IndexFieldDataService fieldDataService) {
         super(shardId, indexSettings);
         this.scriptService = scriptService;
         this.mapperService = mapperService;
-        this.indexCache = indexCache;
+        this.fieldDataService = fieldDataService;
     }
 
     public GetStats stats() {
@@ -213,7 +213,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
                         } else {
                             if (field.contains("_source.")) {
                                 if (searchLookup == null) {
-                                    searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
+                                    searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
                                 }
                                 if (sourceAsMap == null) {
                                     sourceAsMap = SourceLookup.sourceAsMap(source.source);
@@ -236,7 +236,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
                                 }
                             } else {
                                 if (searchLookup == null) {
-                                    searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
+                                    searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
                                     searchLookup.source().setNextSource(source.source);
                                 }
 
@@ -306,7 +306,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
                 Object value = null;
                 if (field.contains("_source.") || field.contains("doc[")) {
                     if (searchLookup == null) {
-                        searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
+                        searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
                     }
                     SearchScript searchScript = scriptService.search(searchLookup, "mvel", field, null);
                     searchScript.setNextReader(docIdAndVersion.reader);
@@ -324,7 +324,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
                     FieldMappers x = docMapper.mappers().smartName(field);
                     if (x == null || !x.mapper().fieldType().stored()) {
                         if (searchLookup == null) {
-                            searchLookup = new SearchLookup(mapperService, indexCache.fieldData(), new String[]{type});
+                            searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type});
                             searchLookup.setNextReader(docIdAndVersion.reader);
                             searchLookup.setNextDocId(docIdAndVersion.docId);
                         }
diff --git a/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java b/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java
index b081cf5024f..fa54ed937f9 100644
--- a/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java
+++ b/src/main/java/org/elasticsearch/index/query/IndexQueryParserService.java
@@ -35,6 +35,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.engine.IndexEngine;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -77,6 +78,8 @@ public class IndexQueryParserService extends AbstractIndexComponent {
 
     final IndexCache indexCache;
 
+    final IndexFieldDataService fieldDataService;
+
     final IndexEngine indexEngine;
 
     private final Map<String, QueryParser> queryParsers;
@@ -90,7 +93,7 @@ public class IndexQueryParserService extends AbstractIndexComponent {
     public IndexQueryParserService(Index index, @IndexSettings Settings indexSettings,
                                    IndicesQueriesRegistry indicesQueriesRegistry,
                                    ScriptService scriptService, AnalysisService analysisService,
-                                   MapperService mapperService, IndexCache indexCache, IndexEngine indexEngine,
+                                   MapperService mapperService, IndexCache indexCache, IndexFieldDataService fieldDataService, IndexEngine indexEngine,
                                    @Nullable SimilarityService similarityService,
                                    @Nullable Map<String, QueryParserFactory> namedQueryParsers,
                                    @Nullable Map<String, FilterParserFactory> namedFilterParsers) {
@@ -100,6 +103,7 @@ public class IndexQueryParserService extends AbstractIndexComponent {
         this.mapperService = mapperService;
         this.similarityService = similarityService;
         this.indexCache = indexCache;
+        this.fieldDataService = fieldDataService;
         this.indexEngine = indexEngine;
 
         this.defaultField = indexSettings.get("index.query.default_field", AllFieldMapper.NAME);
diff --git a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index e9a32f403d1..8b686f88722 100644
--- a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.query;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
-
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
 import org.apache.lucene.queryparser.classic.QueryParserSettings;
 import org.apache.lucene.search.Filter;
@@ -34,6 +33,7 @@ import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
 import org.elasticsearch.index.engine.IndexEngine;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.index.mapper.MapperService;
@@ -130,6 +130,10 @@ public class QueryParseContext {
         return indexQueryParser.indexCache;
     }
 
+    public IndexFieldDataService fieldData() {
+        return indexQueryParser.fieldDataService;
+    }
+
     public String defaultField() {
         return indexQueryParser.defaultField();
     }
@@ -291,7 +295,7 @@ public class QueryParseContext {
             return current.lookup();
         }
         if (lookup == null) {
-            lookup = new SearchLookup(mapperService(), indexCache().fieldData(), null);
+            lookup = new SearchLookup(mapperService(), fieldData(), null);
         }
         return lookup;
     }
diff --git a/src/main/java/org/elasticsearch/script/ScriptService.java b/src/main/java/org/elasticsearch/script/ScriptService.java
index a9973ad84d8..80d8e62aa13 100644
--- a/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -32,7 +32,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.script.mvel.MvelScriptEngineService;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -193,8 +193,8 @@ public class ScriptService extends AbstractComponent {
         return search(compile(lang, script), lookup, vars);
     }
 
-    public SearchScript search(MapperService mapperService, FieldDataCache fieldDataCache, String lang, String script, @Nullable Map<String, Object> vars) {
-        return search(compile(lang, script), new SearchLookup(mapperService, fieldDataCache, null), vars);
+    public SearchScript search(MapperService mapperService, IndexFieldDataService fieldDataService, String lang, String script, @Nullable Map<String, Object> vars) {
+        return search(compile(lang, script), new SearchLookup(mapperService, fieldDataService, null), vars);
     }
 
     public Object execute(CompiledScript compiledScript, Map vars) {
diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 8706f58f337..6284315d454 100644
--- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -546,7 +546,7 @@ public class SearchContext implements Releasable {
     public SearchLookup lookup() {
         // TODO: The types should take into account the parsing context in QueryParserContext...
         if (searchLookup == null) {
-            searchLookup = new SearchLookup(mapperService(), fieldDataCache(), request.types());
+            searchLookup = new SearchLookup(mapperService(), fieldData(), request.types());
         }
         return searchLookup;
     }
diff --git a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
index 3b52c988bb5..7eddf51ca89 100644
--- a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
+++ b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
@@ -20,16 +20,14 @@
 package org.elasticsearch.search.lookup;
 
 import com.google.common.collect.Maps;
-import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
-import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.field.data.DocFieldData;
-import org.elasticsearch.index.field.data.FieldData;
 import org.elasticsearch.index.field.data.NumericDocFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 
@@ -44,24 +42,23 @@ import java.util.Set;
  */
 public class DocLookup implements Map {
 
-    private final Map<String, FieldData> localCacheFieldData = Maps.newHashMapWithExpectedSize(4);
+    private final Map<String, ScriptDocValues> localCacheFieldData = Maps.newHashMapWithExpectedSize(4);
 
     private final MapperService mapperService;
-
-    private final FieldDataCache fieldDataCache;
+    private final IndexFieldDataService fieldDataService;
 
     @Nullable
     private final String[] types;
 
-    private AtomicReader reader;
+    private AtomicReaderContext reader;
 
     private Scorer scorer;
 
     private int docId = -1;
 
-    DocLookup(MapperService mapperService, FieldDataCache fieldDataCache, @Nullable String[] types) {
+    DocLookup(MapperService mapperService, IndexFieldDataService fieldDataService, @Nullable String[] types) {
         this.mapperService = mapperService;
-        this.fieldDataCache = fieldDataCache;
+        this.fieldDataService = fieldDataService;
         this.types = types;
     }
 
@@ -69,15 +66,11 @@ public class DocLookup implements Map {
         return this.mapperService;
     }
 
-    public FieldDataCache fieldDataCache() {
-        return this.fieldDataCache;
-    }
-
     public void setNextReader(AtomicReaderContext context) {
-        if (this.reader == context.reader()) { // if we are called with the same reader, don't invalidate source
+        if (this.reader == context) { // if we are called with the same reader, don't invalidate source
             return;
         }
-        this.reader = context.reader();
+        this.reader = context;
         this.docId = -1;
         localCacheFieldData.clear();
     }
@@ -110,27 +103,24 @@ public class DocLookup implements Map {
     public Object get(Object key) {
         // assume its a string...
         String fieldName = key.toString();
-        FieldData fieldData = localCacheFieldData.get(fieldName);
-        if (fieldData == null) {
+        ScriptDocValues scriptValues = localCacheFieldData.get(fieldName);
+        if (scriptValues == null) {
             FieldMapper mapper = mapperService.smartNameFieldMapper(fieldName, types);
             if (mapper == null) {
                 throw new ElasticSearchIllegalArgumentException("No field found for [" + fieldName + "] in mapping with types " + Arrays.toString(types) + "");
             }
-            try {
-                fieldData = fieldDataCache.cache(mapper.fieldDataType(), reader, mapper.names().indexName());
-            } catch (IOException e) {
-                throw new ElasticSearchException("Failed to load field data for [" + fieldName + "]", e);
-            }
-            localCacheFieldData.put(fieldName, fieldData);
+            scriptValues = fieldDataService.getForField(mapper).load(reader).getScriptValues();
+            localCacheFieldData.put(fieldName, scriptValues);
         }
-        return fieldData.docFieldData(docId);
+        scriptValues.setNextDocId(docId);
+        return scriptValues;
     }
 
     public boolean containsKey(Object key) {
         // assume its a string...
         String fieldName = key.toString();
-        FieldData fieldData = localCacheFieldData.get(fieldName);
-        if (fieldData == null) {
+        ScriptDocValues scriptValues = localCacheFieldData.get(fieldName);
+        if (scriptValues == null) {
             FieldMapper mapper = mapperService.smartNameFieldMapper(fieldName, types);
             if (mapper == null) {
                 return false;
diff --git a/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
index efa151b5cd5..9a8036038fb 100644
--- a/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
+++ b/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
@@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMap;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
 
 /**
@@ -39,8 +39,8 @@ public class SearchLookup {
 
     final ImmutableMap<String, Object> asMap;
 
-    public SearchLookup(MapperService mapperService, FieldDataCache fieldDataCache, @Nullable String[] types) {
-        docMap = new DocLookup(mapperService, fieldDataCache, types);
+    public SearchLookup(MapperService mapperService, IndexFieldDataService fieldDataService, @Nullable String[] types) {
+        docMap = new DocLookup(mapperService, fieldDataService, types);
         sourceLookup = new SourceLookup();
         fieldsLookup = new FieldsLookup(mapperService, types);
         asMap = ImmutableMap.<String, Object>of("doc", docMap, "_doc", docMap, "_source", sourceLookup, "_fields", fieldsLookup);

From de013babf847bdedf89bf9744d7a5ab1bb807b40 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 14:35:24 +0100
Subject: [PATCH 29/35] move geo filters and numeric range to use new field
 data

---
 .../index/fielddata/FieldDataType.java        |   4 +
 .../index/mapper/core/ByteFieldMapper.java    |   7 +-
 .../index/mapper/core/DateFieldMapper.java    |   7 +-
 .../index/mapper/core/DoubleFieldMapper.java  |   7 +-
 .../index/mapper/core/FloatFieldMapper.java   |   7 +-
 .../index/mapper/core/IntegerFieldMapper.java |   7 +-
 .../index/mapper/core/LongFieldMapper.java    |   7 +-
 .../index/mapper/core/NumberFieldMapper.java  |   4 +-
 .../index/mapper/core/ShortFieldMapper.java   |   7 +-
 .../index/mapper/geo/GeoPointFieldMapper.java |   4 +
 .../mapper/internal/BoostFieldMapper.java     |   7 +-
 .../index/mapper/internal/UidFieldMapper.java |   2 +-
 .../index/mapper/ip/IpFieldMapper.java        |   7 +-
 .../query/GeoBoundingBoxFilterParser.java     |   6 +-
 .../index/query/GeoDistanceFilterParser.java  |   9 +-
 .../query/GeoDistanceRangeFilterParser.java   |   8 +-
 .../index/query/GeoPolygonFilterParser.java   |   8 +-
 .../index/query/NumericRangeFilterParser.java |   2 +-
 .../index/query/QueryParseContext.java        |   4 +
 .../search/NumericRangeFieldDataFilter.java   | 176 ++++++------------
 .../index/search/geo/GeoDistanceFilter.java   |  57 +++---
 .../search/geo/GeoDistanceRangeFilter.java    |  59 +++---
 .../index/search/geo/GeoPolygonFilter.java    |  46 +++--
 .../geo/InMemoryGeoBoundingBoxFilter.java     |  83 ++++-----
 .../search/internal/SearchContext.java        |   5 -
 25 files changed, 239 insertions(+), 301 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java b/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
index f354a154c97..e53f13a35fb 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java
@@ -35,6 +35,10 @@ public class FieldDataType {
         this(type, null, ImmutableMap.<String, String>of());
     }
 
+    public FieldDataType(String type, String format) {
+        this(type, format, ImmutableMap.<String, String>of());
+    }
+
     public FieldDataType(String type, @Nullable String format, ImmutableMap<String, String> options) {
         this.type = type;
         this.format = format;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
index 386907e6800..3d75355912b 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
@@ -37,9 +37,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
@@ -229,8 +230,8 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newByteRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newByteRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 045fa403fe5..0982cd6b41e 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -40,9 +40,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericDateAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
@@ -310,9 +311,9 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
         long now = context == null ? System.currentTimeMillis() : context.nowInMillis();
-        return NumericRangeFieldDataFilter.newLongRange(fieldDataCache, names.indexName(),
+        return NumericRangeFieldDataFilter.newLongRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : dateMathParser.parse(convertToString(lowerTerm), now),
                 upperTerm == null ? null : (includeUpper && parseUpperInclusive) ? dateMathParser.parseUpperInclusive(convertToString(upperTerm), now) : dateMathParser.parse(convertToString(upperTerm), now),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
index 3e7ff3c4649..4e2197471dc 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
@@ -37,9 +37,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericDoubleAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
@@ -228,8 +229,8 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newDoubleRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newDoubleRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
index 3f3253b9af0..bc8f4dcceb2 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
@@ -38,9 +38,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericFloatAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
@@ -222,8 +223,8 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newFloatRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newFloatRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
index 9bbe52b1ee0..2057e08bbd9 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
@@ -38,9 +38,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
@@ -228,8 +229,8 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newIntRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newIntRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
index 1055acfc93c..0c1858b40a9 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
@@ -38,9 +38,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericLongAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
@@ -228,8 +229,8 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newLongRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newLongRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index 4f9b0df2f35..2f44f9fc241 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -30,9 +30,9 @@ import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -242,7 +242,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
     /**
      * A range filter based on the field data cache.
      */
-    public abstract Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context);
+    public abstract Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context);
 
     /**
      * Override the default behavior (to return the string, and return the actual Number instance).
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
index 7c410ad093c..f76209512a6 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
@@ -38,9 +38,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
@@ -232,8 +233,8 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newShortRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newShortRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
index 72d112e9dff..d3caf0cf509 100644
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
@@ -287,6 +287,10 @@ public class GeoPointFieldMapper implements Mapper, ArrayValueMapperParser {
         return lonMapper;
     }
 
+    public GeoStringFieldMapper stringMapper() {
+        return this.geoStringMapper;
+    }
+
     public boolean isEnableLatLon() {
         return enableLatLon;
     }
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
index a5c21775663..31ce24a80a1 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
@@ -35,9 +35,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericFloatAnalyzer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.FloatFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
@@ -211,8 +212,8 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newFloatRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newFloatRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index fe95914bb1b..2c6341157d1 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -129,7 +129,7 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
 
     @Override
     public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        return new FieldDataType("string");
+        return new FieldDataType("string", "paged_bytes");
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 321991d1d78..f2d07688c7f 100644
--- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -35,9 +35,10 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericAnalyzer;
 import org.elasticsearch.index.analysis.NumericTokenizer;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
 import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.LongFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
@@ -249,8 +250,8 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
     }
 
     @Override
-    public Filter rangeFilter(FieldDataCache fieldDataCache, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
-        return NumericRangeFieldDataFilter.newLongRange(fieldDataCache, names.indexName(),
+    public Filter rangeFilter(IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) {
+        return NumericRangeFieldDataFilter.newLongRange((IndexNumericFieldData) fieldData.getForField(this),
                 lowerTerm == null ? null : parseValue(lowerTerm),
                 upperTerm == null ? null : parseValue(upperTerm),
                 includeLower, includeUpper);
diff --git a/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java b/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java
index 432d588bab8..3bf595be882 100644
--- a/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java
+++ b/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java
@@ -23,6 +23,7 @@ import org.apache.lucene.search.Filter;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
@@ -174,13 +175,12 @@ public class GeoBoundingBoxFilterParser implements FilterParser {
         }
         GeoPointFieldMapper geoMapper = ((GeoPointFieldMapper.GeoStringFieldMapper) mapper).geoMapper();
 
-        fieldName = mapper.names().indexName();
-
         Filter filter;
         if ("indexed".equals(type)) {
             filter = IndexedGeoBoundingBoxFilter.create(topLeft, bottomRight, geoMapper);
         } else if ("memory".equals(type)) {
-            filter = new InMemoryGeoBoundingBoxFilter(topLeft, bottomRight, fieldName, parseContext.indexCache().fieldData());
+            IndexGeoPointFieldData indexFieldData = parseContext.fieldData().getForField(mapper);
+            filter = new InMemoryGeoBoundingBoxFilter(topLeft, bottomRight, indexFieldData);
         } else {
             throw new QueryParsingException(parseContext.index(), "geo bounding box type [" + type + "] not supported, either 'indexed' or 'memory' are allowed");
         }
diff --git a/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java b/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java
index 0b216d85821..75ecf780f87 100644
--- a/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java
+++ b/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java
@@ -24,9 +24,9 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.search.geo.*;
 
@@ -177,13 +177,14 @@ public class GeoDistanceFilterParser implements FilterParser {
             throw new QueryParsingException(parseContext.index(), "failed to find geo_point field [" + fieldName + "]");
         }
         FieldMapper mapper = smartMappers.mapper();
-        if (mapper.fieldDataType() != GeoPointFieldDataType.TYPE) {
+        if (!(mapper instanceof GeoPointFieldMapper.GeoStringFieldMapper)) {
             throw new QueryParsingException(parseContext.index(), "field [" + fieldName + "] is not a geo_point field");
         }
         GeoPointFieldMapper geoMapper = ((GeoPointFieldMapper.GeoStringFieldMapper) mapper).geoMapper();
-        fieldName = mapper.names().indexName();
 
-        Filter filter = new GeoDistanceFilter(lat, lon, distance, geoDistance, fieldName, geoMapper, parseContext.indexCache().fieldData(), optimizeBbox);
+
+        IndexGeoPointFieldData indexFieldData = parseContext.fieldData().getForField(mapper);
+        Filter filter = new GeoDistanceFilter(lat, lon, distance, geoDistance, indexFieldData, geoMapper, optimizeBbox);
         if (cache) {
             filter = parseContext.cacheFilter(filter, cacheKey);
         }
diff --git a/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java
index 296ef8b2342..be4f8ffc5f5 100644
--- a/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java
+++ b/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java
@@ -24,9 +24,9 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.search.geo.*;
 
@@ -233,13 +233,13 @@ public class GeoDistanceRangeFilterParser implements FilterParser {
             throw new QueryParsingException(parseContext.index(), "failed to find geo_point field [" + fieldName + "]");
         }
         FieldMapper mapper = smartMappers.mapper();
-        if (mapper.fieldDataType() != GeoPointFieldDataType.TYPE) {
+        if (!(mapper instanceof GeoPointFieldMapper.GeoStringFieldMapper)) {
             throw new QueryParsingException(parseContext.index(), "field [" + fieldName + "] is not a geo_point field");
         }
         GeoPointFieldMapper geoMapper = ((GeoPointFieldMapper.GeoStringFieldMapper) mapper).geoMapper();
-        fieldName = mapper.names().indexName();
 
-        Filter filter = new GeoDistanceRangeFilter(lat, lon, from, to, includeLower, includeUpper, geoDistance, fieldName, geoMapper, parseContext.indexCache().fieldData(), optimizeBbox);
+        IndexGeoPointFieldData indexFieldData = parseContext.fieldData().getForField(mapper);
+        Filter filter = new GeoDistanceRangeFilter(lat, lon, from, to, includeLower, includeUpper, geoDistance, geoMapper, indexFieldData, optimizeBbox);
         if (cache) {
             filter = parseContext.cacheFilter(filter, cacheKey);
         }
diff --git a/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java b/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java
index e8417e2f2a4..05150896374 100644
--- a/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java
+++ b/src/main/java/org/elasticsearch/index/query/GeoPolygonFilterParser.java
@@ -24,9 +24,9 @@ import org.apache.lucene.search.Filter;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.search.geo.GeoHashUtils;
 import org.elasticsearch.index.search.geo.GeoPolygonFilter;
@@ -172,12 +172,12 @@ public class GeoPolygonFilterParser implements FilterParser {
             throw new QueryParsingException(parseContext.index(), "failed to find geo_point field [" + fieldName + "]");
         }
         FieldMapper mapper = smartMappers.mapper();
-        if (mapper.fieldDataType() != GeoPointFieldDataType.TYPE) {
+        if (!(mapper instanceof GeoPointFieldMapper.GeoStringFieldMapper)) {
             throw new QueryParsingException(parseContext.index(), "field [" + fieldName + "] is not a geo_point field");
         }
-        fieldName = mapper.names().indexName();
 
-        Filter filter = new GeoPolygonFilter(points.toArray(new Point[points.size()]), fieldName, parseContext.indexCache().fieldData());
+        IndexGeoPointFieldData indexFieldData = parseContext.fieldData().getForField(mapper);
+        Filter filter = new GeoPolygonFilter(points.toArray(new Point[points.size()]), indexFieldData);
         if (cache) {
             filter = parseContext.cacheFilter(filter, cacheKey);
         }
diff --git a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java
index 737a70d92f9..1729e14a1e9 100644
--- a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java
+++ b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java
@@ -119,7 +119,7 @@ public class NumericRangeFilterParser implements FilterParser {
         if (!(mapper instanceof NumberFieldMapper)) {
             throw new QueryParsingException(parseContext.index(), "Field [" + fieldName + "] is not a numeric type");
         }
-        Filter filter = ((NumberFieldMapper) mapper).rangeFilter(parseContext.indexCache().fieldData(), from, to, includeLower, includeUpper, parseContext);
+        Filter filter = ((NumberFieldMapper) mapper).rangeFilter(parseContext.fieldData(), from, to, includeLower, includeUpper, parseContext);
 
         if (cache) {
             filter = parseContext.cacheFilter(filter, cacheKey);
diff --git a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index 8b686f88722..3730ece129e 100644
--- a/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -269,6 +269,10 @@ public class QueryParseContext {
         return indexQueryParser.mapperService.smartName(name, getTypes());
     }
 
+    public FieldMapper smartNameFieldMapper(String name) {
+        return indexQueryParser.mapperService.smartNameFieldMapper(name, getTypes());
+    }
+
     public MapperService.SmartNameObjectMapper smartObjectMapper(String name) {
         return indexQueryParser.mapperService.smartNameObjectMapper(name, getTypes());
     }
diff --git a/src/main/java/org/elasticsearch/index/search/NumericRangeFieldDataFilter.java b/src/main/java/org/elasticsearch/index/search/NumericRangeFieldDataFilter.java
index 89aa2e4beba..cb6d6995611 100644
--- a/src/main/java/org/elasticsearch/index/search/NumericRangeFieldDataFilter.java
+++ b/src/main/java/org/elasticsearch/index/search/NumericRangeFieldDataFilter.java
@@ -25,14 +25,7 @@ import org.apache.lucene.search.Filter;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.lucene.docset.MatchDocIdSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.bytes.ByteFieldData;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.elasticsearch.index.field.data.floats.FloatFieldData;
-import org.elasticsearch.index.field.data.ints.IntFieldData;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.elasticsearch.index.field.data.shorts.ShortFieldData;
+import org.elasticsearch.index.fielddata.*;
 
 import java.io.IOException;
 
@@ -41,16 +34,14 @@ import java.io.IOException;
  * expense of loading numeric values of the field to memory using {@link org.elasticsearch.index.cache.field.data.FieldDataCache}.
  */
 public abstract class NumericRangeFieldDataFilter<T> extends Filter {
-    // LUCENE 4 UPGRADE: this filter doesn't respect acceptDocs yet!
-    final FieldDataCache fieldDataCache;
-    final String field;
+    final IndexNumericFieldData indexFieldData;
     final T lowerVal;
     final T upperVal;
     final boolean includeLower;
     final boolean includeUpper;
 
     public String getField() {
-        return field;
+        return indexFieldData.getFieldNames().indexName();
     }
 
     public T getLowerVal() {
@@ -69,9 +60,8 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
         return includeUpper;
     }
 
-    protected NumericRangeFieldDataFilter(FieldDataCache fieldDataCache, String field, T lowerVal, T upperVal, boolean includeLower, boolean includeUpper) {
-        this.fieldDataCache = fieldDataCache;
-        this.field = field;
+    protected NumericRangeFieldDataFilter(IndexNumericFieldData indexFieldData, T lowerVal, T upperVal, boolean includeLower, boolean includeUpper) {
+        this.indexFieldData = indexFieldData;
         this.lowerVal = lowerVal;
         this.upperVal = upperVal;
         this.includeLower = includeLower;
@@ -80,7 +70,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
     @Override
     public final String toString() {
-        final StringBuilder sb = new StringBuilder(field).append(":");
+        final StringBuilder sb = new StringBuilder(indexFieldData.getFieldNames().indexName()).append(":");
         return sb.append(includeLower ? '[' : '{')
                 .append((lowerVal == null) ? "*" : lowerVal.toString())
                 .append(" TO ")
@@ -95,7 +85,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
         if (!(o instanceof NumericRangeFieldDataFilter)) return false;
         NumericRangeFieldDataFilter other = (NumericRangeFieldDataFilter) o;
 
-        if (!this.field.equals(other.field)
+        if (!this.indexFieldData.getFieldNames().indexName().equals(other.indexFieldData.getFieldNames().indexName())
                 || this.includeLower != other.includeLower
                 || this.includeUpper != other.includeUpper
                 ) {
@@ -108,7 +98,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
     @Override
     public final int hashCode() {
-        int h = field.hashCode();
+        int h = indexFieldData.getFieldNames().indexName().hashCode();
         h ^= (lowerVal != null) ? lowerVal.hashCode() : 550356204;
         h = (h << 1) | (h >>> 31);  // rotate to distinguish lower from upper
         h ^= (upperVal != null) ? upperVal.hashCode() : -1674416163;
@@ -116,8 +106,8 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
         return h;
     }
 
-    public static NumericRangeFieldDataFilter<Byte> newByteRange(FieldDataCache fieldDataCache, String field, Byte lowerVal, Byte upperVal, boolean includeLower, boolean includeUpper) {
-        return new NumericRangeFieldDataFilter<Byte>(fieldDataCache, field, lowerVal, upperVal, includeLower, includeUpper) {
+    public static NumericRangeFieldDataFilter<Byte> newByteRange(IndexNumericFieldData indexFieldData, Byte lowerVal, Byte upperVal, boolean includeLower, boolean includeUpper) {
+        return new NumericRangeFieldDataFilter<Byte>(indexFieldData, lowerVal, upperVal, includeLower, includeUpper) {
             @Override
             public DocIdSet getDocIdSet(AtomicReaderContext ctx, Bits acceptedDocs) throws IOException {
                 final byte inclusiveLowerPoint, inclusiveUpperPoint;
@@ -141,7 +131,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
                 if (inclusiveLowerPoint > inclusiveUpperPoint)
                     return null;
 
-                final ByteFieldData fieldData = (ByteFieldData) this.fieldDataCache.cache(FieldDataType.DefaultTypes.BYTE, ctx.reader(), field);
+                final ByteValues values = indexFieldData.load(ctx).getByteValues();
                 return new MatchDocIdSet(ctx.reader().maxDoc(), acceptedDocs) {
 
                     @Override
@@ -151,21 +141,14 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
                     @Override
                     protected boolean matchDoc(int doc) {
-                        if (!fieldData.hasValue(doc)) {
-                            return false;
-                        }
-                        if (fieldData.multiValued()) {
-                            byte[] values = fieldData.values(doc);
-                            for (byte value : values) {
-                                if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
-                                    return true;
-                                }
+                        ByteValues.Iter iter = values.getIter(doc);
+                        while (iter.hasNext()) {
+                            byte value = iter.next();
+                            if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
+                                return true;
                             }
-                            return false;
-                        } else {
-                            byte value = fieldData.value(doc);
-                            return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
                         }
+                        return false;
                     }
                 };
             }
@@ -173,8 +156,8 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
     }
 
 
-    public static NumericRangeFieldDataFilter<Short> newShortRange(FieldDataCache fieldDataCache, String field, Short lowerVal, Short upperVal, boolean includeLower, boolean includeUpper) {
-        return new NumericRangeFieldDataFilter<Short>(fieldDataCache, field, lowerVal, upperVal, includeLower, includeUpper) {
+    public static NumericRangeFieldDataFilter<Short> newShortRange(IndexNumericFieldData indexFieldData, Short lowerVal, Short upperVal, boolean includeLower, boolean includeUpper) {
+        return new NumericRangeFieldDataFilter<Short>(indexFieldData, lowerVal, upperVal, includeLower, includeUpper) {
             @Override
             public DocIdSet getDocIdSet(AtomicReaderContext ctx, Bits acceptedDocs) throws IOException {
                 final short inclusiveLowerPoint, inclusiveUpperPoint;
@@ -198,7 +181,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
                 if (inclusiveLowerPoint > inclusiveUpperPoint)
                     return null;
 
-                final ShortFieldData fieldData = (ShortFieldData) this.fieldDataCache.cache(FieldDataType.DefaultTypes.SHORT, ctx.reader(), field);
+                final ShortValues values = indexFieldData.load(ctx).getShortValues();
                 return new MatchDocIdSet(ctx.reader().maxDoc(), acceptedDocs) {
 
                     @Override
@@ -208,29 +191,22 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
                     @Override
                     protected boolean matchDoc(int doc) {
-                        if (!fieldData.hasValue(doc)) {
-                            return false;
-                        }
-                        if (fieldData.multiValued()) {
-                            short[] values = fieldData.values(doc);
-                            for (short value : values) {
-                                if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
-                                    return true;
-                                }
+                        ShortValues.Iter iter = values.getIter(doc);
+                        while (iter.hasNext()) {
+                            short value = iter.next();
+                            if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
+                                return true;
                             }
-                            return false;
-                        } else {
-                            short value = fieldData.value(doc);
-                            return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
                         }
+                        return false;
                     }
                 };
             }
         };
     }
 
-    public static NumericRangeFieldDataFilter<Integer> newIntRange(FieldDataCache fieldDataCache, String field, Integer lowerVal, Integer upperVal, boolean includeLower, boolean includeUpper) {
-        return new NumericRangeFieldDataFilter<Integer>(fieldDataCache, field, lowerVal, upperVal, includeLower, includeUpper) {
+    public static NumericRangeFieldDataFilter<Integer> newIntRange(IndexNumericFieldData indexFieldData, Integer lowerVal, Integer upperVal, boolean includeLower, boolean includeUpper) {
+        return new NumericRangeFieldDataFilter<Integer>(indexFieldData, lowerVal, upperVal, includeLower, includeUpper) {
             @Override
             public DocIdSet getDocIdSet(AtomicReaderContext ctx, Bits acceptedDocs) throws IOException {
                 final int inclusiveLowerPoint, inclusiveUpperPoint;
@@ -254,7 +230,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
                 if (inclusiveLowerPoint > inclusiveUpperPoint)
                     return null;
 
-                final IntFieldData fieldData = (IntFieldData) this.fieldDataCache.cache(FieldDataType.DefaultTypes.INT, ctx.reader(), field);
+                final IntValues values = indexFieldData.load(ctx).getIntValues();
                 return new MatchDocIdSet(ctx.reader().maxDoc(), acceptedDocs) {
 
                     @Override
@@ -264,29 +240,22 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
                     @Override
                     protected boolean matchDoc(int doc) {
-                        if (!fieldData.hasValue(doc)) {
-                            return false;
-                        }
-                        if (fieldData.multiValued()) {
-                            int[] values = fieldData.values(doc);
-                            for (int value : values) {
-                                if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
-                                    return true;
-                                }
+                        IntValues.Iter iter = values.getIter(doc);
+                        while (iter.hasNext()) {
+                            int value = iter.next();
+                            if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
+                                return true;
                             }
-                            return false;
-                        } else {
-                            int value = fieldData.value(doc);
-                            return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
                         }
+                        return false;
                     }
                 };
             }
         };
     }
 
-    public static NumericRangeFieldDataFilter<Long> newLongRange(FieldDataCache fieldDataCache, String field, Long lowerVal, Long upperVal, boolean includeLower, boolean includeUpper) {
-        return new NumericRangeFieldDataFilter<Long>(fieldDataCache, field, lowerVal, upperVal, includeLower, includeUpper) {
+    public static NumericRangeFieldDataFilter<Long> newLongRange(IndexNumericFieldData indexFieldData, Long lowerVal, Long upperVal, boolean includeLower, boolean includeUpper) {
+        return new NumericRangeFieldDataFilter<Long>(indexFieldData, lowerVal, upperVal, includeLower, includeUpper) {
             @Override
             public DocIdSet getDocIdSet(AtomicReaderContext ctx, Bits acceptedDocs) throws IOException {
                 final long inclusiveLowerPoint, inclusiveUpperPoint;
@@ -310,7 +279,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
                 if (inclusiveLowerPoint > inclusiveUpperPoint)
                     return null;
 
-                final LongFieldData fieldData = (LongFieldData) this.fieldDataCache.cache(FieldDataType.DefaultTypes.LONG, ctx.reader(), field);
+                final LongValues values = indexFieldData.load(ctx).getLongValues();
                 return new MatchDocIdSet(ctx.reader().maxDoc(), acceptedDocs) {
 
                     @Override
@@ -320,29 +289,22 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
                     @Override
                     protected boolean matchDoc(int doc) {
-                        if (!fieldData.hasValue(doc)) {
-                            return false;
-                        }
-                        if (fieldData.multiValued()) {
-                            long[] values = fieldData.values(doc);
-                            for (long value : values) {
-                                if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
-                                    return true;
-                                }
+                        LongValues.Iter iter = values.getIter(doc);
+                        while (iter.hasNext()) {
+                            long value = iter.next();
+                            if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
+                                return true;
                             }
-                            return false;
-                        } else {
-                            long value = fieldData.value(doc);
-                            return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
                         }
+                        return false;
                     }
                 };
             }
         };
     }
 
-    public static NumericRangeFieldDataFilter<Float> newFloatRange(FieldDataCache fieldDataCache, String field, Float lowerVal, Float upperVal, boolean includeLower, boolean includeUpper) {
-        return new NumericRangeFieldDataFilter<Float>(fieldDataCache, field, lowerVal, upperVal, includeLower, includeUpper) {
+    public static NumericRangeFieldDataFilter<Float> newFloatRange(IndexNumericFieldData indexFieldData, Float lowerVal, Float upperVal, boolean includeLower, boolean includeUpper) {
+        return new NumericRangeFieldDataFilter<Float>(indexFieldData, lowerVal, upperVal, includeLower, includeUpper) {
             @Override
             public DocIdSet getDocIdSet(AtomicReaderContext ctx, Bits acceptedDocs) throws IOException {
                 // we transform the floating point numbers to sortable integers
@@ -370,7 +332,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
                 if (inclusiveLowerPoint > inclusiveUpperPoint)
                     return null;
 
-                final FloatFieldData fieldData = (FloatFieldData) this.fieldDataCache.cache(FieldDataType.DefaultTypes.FLOAT, ctx.reader(), field);
+                final FloatValues values = indexFieldData.load(ctx).getFloatValues();
                 return new MatchDocIdSet(ctx.reader().maxDoc(), acceptedDocs) {
 
                     @Override
@@ -380,29 +342,22 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
                     @Override
                     protected boolean matchDoc(int doc) {
-                        if (!fieldData.hasValue(doc)) {
-                            return false;
-                        }
-                        if (fieldData.multiValued()) {
-                            float[] values = fieldData.values(doc);
-                            for (float value : values) {
-                                if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
-                                    return true;
-                                }
+                        FloatValues.Iter iter = values.getIter(doc);
+                        while (iter.hasNext()) {
+                            float value = iter.next();
+                            if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
+                                return true;
                             }
-                            return false;
-                        } else {
-                            float value = fieldData.value(doc);
-                            return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
                         }
+                        return false;
                     }
                 };
             }
         };
     }
 
-    public static NumericRangeFieldDataFilter<Double> newDoubleRange(FieldDataCache fieldDataCache, String field, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper) {
-        return new NumericRangeFieldDataFilter<Double>(fieldDataCache, field, lowerVal, upperVal, includeLower, includeUpper) {
+    public static NumericRangeFieldDataFilter<Double> newDoubleRange(IndexNumericFieldData indexFieldData, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper) {
+        return new NumericRangeFieldDataFilter<Double>(indexFieldData, lowerVal, upperVal, includeLower, includeUpper) {
             @Override
             public DocIdSet getDocIdSet(AtomicReaderContext ctx, Bits acceptedDocs) throws IOException {
                 // we transform the floating point numbers to sortable integers
@@ -430,7 +385,7 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
                 if (inclusiveLowerPoint > inclusiveUpperPoint)
                     return null;
 
-                final DoubleFieldData fieldData = (DoubleFieldData) this.fieldDataCache.cache(FieldDataType.DefaultTypes.DOUBLE, ctx.reader(), field);
+                final DoubleValues values = indexFieldData.load(ctx).getDoubleValues();
                 return new MatchDocIdSet(ctx.reader().maxDoc(), acceptedDocs) {
 
                     @Override
@@ -440,21 +395,14 @@ public abstract class NumericRangeFieldDataFilter<T> extends Filter {
 
                     @Override
                     protected boolean matchDoc(int doc) {
-                        if (!fieldData.hasValue(doc)) {
-                            return false;
-                        }
-                        if (fieldData.multiValued()) {
-                            double[] values = fieldData.values(doc);
-                            for (double value : values) {
-                                if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
-                                    return true;
-                                }
+                        DoubleValues.Iter iter = values.getIter(doc);
+                        while (iter.hasNext()) {
+                            double value = iter.next();
+                            if (value >= inclusiveLowerPoint && value <= inclusiveUpperPoint) {
+                                return true;
                             }
-                            return false;
-                        } else {
-                            double value = fieldData.value(doc);
-                            return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
                         }
+                        return false;
                     }
                 };
             }
diff --git a/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java b/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java
index f00087ec739..648f39023aa 100644
--- a/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java
+++ b/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java
@@ -29,9 +29,9 @@ import org.elasticsearch.common.lucene.docset.AndDocIdSet;
 import org.elasticsearch.common.lucene.docset.DocIdSets;
 import org.elasticsearch.common.lucene.docset.MatchDocIdSet;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 
 import java.io.IOException;
@@ -48,22 +48,19 @@ public class GeoDistanceFilter extends Filter {
 
     private final GeoDistance geoDistance;
 
-    private final String fieldName;
-
-    private final FieldDataCache fieldDataCache;
+    private final IndexGeoPointFieldData indexFieldData;
 
     private final GeoDistance.FixedSourceDistance fixedSourceDistance;
     private GeoDistance.DistanceBoundingCheck distanceBoundingCheck;
     private final Filter boundingBoxFilter;
 
-    public GeoDistanceFilter(double lat, double lon, double distance, GeoDistance geoDistance, String fieldName, GeoPointFieldMapper mapper, FieldDataCache fieldDataCache,
+    public GeoDistanceFilter(double lat, double lon, double distance, GeoDistance geoDistance, IndexGeoPointFieldData indexFieldData, GeoPointFieldMapper mapper,
                              String optimizeBbox) {
         this.lat = lat;
         this.lon = lon;
         this.distance = distance;
         this.geoDistance = geoDistance;
-        this.fieldName = fieldName;
-        this.fieldDataCache = fieldDataCache;
+        this.indexFieldData = indexFieldData;
 
         this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, DistanceUnit.MILES);
         if (optimizeBbox != null && !"none".equals(optimizeBbox)) {
@@ -99,7 +96,7 @@ public class GeoDistanceFilter extends Filter {
     }
 
     public String fieldName() {
-        return fieldName;
+        return indexFieldData.getFieldNames().indexName();
     }
 
     @Override
@@ -111,8 +108,8 @@ public class GeoDistanceFilter extends Filter {
                 return null;
             }
         }
-        final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, context.reader(), fieldName);
-        GeoDistanceDocSet distDocSet = new GeoDistanceDocSet(context.reader().maxDoc(), acceptedDocs, fieldData, fixedSourceDistance, distanceBoundingCheck, distance);
+        final GeoPointValues values = indexFieldData.load(context).getGeoPointValues();
+        GeoDistanceDocSet distDocSet = new GeoDistanceDocSet(context.reader().maxDoc(), acceptedDocs, values, fixedSourceDistance, distanceBoundingCheck, distance);
         if (boundingBoxDocSet == null) {
             return distDocSet;
         } else {
@@ -130,7 +127,8 @@ public class GeoDistanceFilter extends Filter {
         if (Double.compare(filter.distance, distance) != 0) return false;
         if (Double.compare(filter.lat, lat) != 0) return false;
         if (Double.compare(filter.lon, lon) != 0) return false;
-        if (fieldName != null ? !fieldName.equals(filter.fieldName) : filter.fieldName != null) return false;
+        if (!indexFieldData.getFieldNames().indexName().equals(filter.indexFieldData.getFieldNames().indexName()))
+            return false;
         if (geoDistance != filter.geoDistance) return false;
 
         return true;
@@ -138,7 +136,7 @@ public class GeoDistanceFilter extends Filter {
 
     @Override
     public String toString() {
-        return "GeoDistanceFilter(" + fieldName + ", " + geoDistance + ", " + distance + ", " + lat + ", " + lon + ")";
+        return "GeoDistanceFilter(" + indexFieldData.getFieldNames().indexName() + ", " + geoDistance + ", " + distance + ", " + lat + ", " + lon + ")";
     }
 
     @Override
@@ -152,20 +150,20 @@ public class GeoDistanceFilter extends Filter {
         temp = distance != +0.0d ? Double.doubleToLongBits(distance) : 0L;
         result = 31 * result + (int) (temp ^ (temp >>> 32));
         result = 31 * result + (geoDistance != null ? geoDistance.hashCode() : 0);
-        result = 31 * result + (fieldName != null ? fieldName.hashCode() : 0);
+        result = 31 * result + indexFieldData.getFieldNames().indexName().hashCode();
         return result;
     }
 
     public static class GeoDistanceDocSet extends MatchDocIdSet {
         private final double distance; // in miles
-        private final GeoPointFieldData fieldData;
+        private final GeoPointValues values;
         private final GeoDistance.FixedSourceDistance fixedSourceDistance;
         private final GeoDistance.DistanceBoundingCheck distanceBoundingCheck;
 
-        public GeoDistanceDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointFieldData fieldData, GeoDistance.FixedSourceDistance fixedSourceDistance, GeoDistance.DistanceBoundingCheck distanceBoundingCheck,
+        public GeoDistanceDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointValues values, GeoDistance.FixedSourceDistance fixedSourceDistance, GeoDistance.DistanceBoundingCheck distanceBoundingCheck,
                                  double distance) {
             super(maxDoc, acceptDocs);
-            this.fieldData = fieldData;
+            this.values = values;
             this.fixedSourceDistance = fixedSourceDistance;
             this.distanceBoundingCheck = distanceBoundingCheck;
             this.distance = distance;
@@ -178,18 +176,16 @@ public class GeoDistanceFilter extends Filter {
 
         @Override
         protected boolean matchDoc(int doc) {
-            if (!fieldData.hasValue(doc)) {
+            if (!values.hasValue(doc)) {
                 return false;
             }
 
-            if (fieldData.multiValued()) {
-                double[] lats = fieldData.latValues(doc);
-                double[] lons = fieldData.lonValues(doc);
-                for (int i = 0; i < lats.length; i++) {
-                    double lat = lats[i];
-                    double lon = lons[i];
-                    if (distanceBoundingCheck.isWithin(lat, lon)) {
-                        double d = fixedSourceDistance.calculate(lat, lon);
+            if (values.isMultiValued()) {
+                GeoPointValues.Iter iter = values.getIter(doc);
+                while (iter.hasNext()) {
+                    GeoPoint point = iter.next();
+                    if (distanceBoundingCheck.isWithin(point.lat(), point.lon())) {
+                        double d = fixedSourceDistance.calculate(point.lat(), point.lon());
                         if (d < distance) {
                             return true;
                         }
@@ -197,10 +193,9 @@ public class GeoDistanceFilter extends Filter {
                 }
                 return false;
             } else {
-                double lat = fieldData.latValue(doc);
-                double lon = fieldData.lonValue(doc);
-                if (distanceBoundingCheck.isWithin(lat, lon)) {
-                    double d = fixedSourceDistance.calculate(lat, lon);
+                GeoPoint point = values.getValue(doc);
+                if (distanceBoundingCheck.isWithin(point.lat(), point.lon())) {
+                    double d = fixedSourceDistance.calculate(point.lat(), point.lon());
                     return d < distance;
                 }
             }
diff --git a/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java b/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java
index 71faf7c0291..e8c2b546ccc 100644
--- a/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java
+++ b/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java
@@ -30,9 +30,9 @@ import org.elasticsearch.common.lucene.docset.AndDocIdSet;
 import org.elasticsearch.common.lucene.docset.DocIdSets;
 import org.elasticsearch.common.lucene.docset.MatchDocIdSet;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 
 import java.io.IOException;
@@ -53,17 +53,14 @@ public class GeoDistanceRangeFilter extends Filter {
     private GeoDistance.DistanceBoundingCheck distanceBoundingCheck;
     private final Filter boundingBoxFilter;
 
-    private final String fieldName;
+    private final IndexGeoPointFieldData indexFieldData;
 
-    private final FieldDataCache fieldDataCache;
-
-    public GeoDistanceRangeFilter(double lat, double lon, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper, GeoDistance geoDistance, String fieldName, GeoPointFieldMapper mapper, FieldDataCache fieldDataCache,
+    public GeoDistanceRangeFilter(double lat, double lon, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper, GeoDistance geoDistance, GeoPointFieldMapper mapper, IndexGeoPointFieldData indexFieldData,
                                   String optimizeBbox) {
         this.lat = lat;
         this.lon = lon;
         this.geoDistance = geoDistance;
-        this.fieldName = fieldName;
-        this.fieldDataCache = fieldDataCache;
+        this.indexFieldData = indexFieldData;
 
         this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, DistanceUnit.MILES);
 
@@ -114,10 +111,6 @@ public class GeoDistanceRangeFilter extends Filter {
         return geoDistance;
     }
 
-    public String fieldName() {
-        return fieldName;
-    }
-
     @Override
     public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptedDocs) throws IOException {
         DocIdSet boundingBoxDocSet = null;
@@ -127,8 +120,8 @@ public class GeoDistanceRangeFilter extends Filter {
                 return null;
             }
         }
-        final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, context.reader(), fieldName);
-        GeoDistanceRangeDocSet distDocSet = new GeoDistanceRangeDocSet(context.reader().maxDoc(), acceptedDocs, fieldData, fixedSourceDistance, distanceBoundingCheck, inclusiveLowerPoint, inclusiveUpperPoint);
+        GeoPointValues values = indexFieldData.load(context).getGeoPointValues();
+        GeoDistanceRangeDocSet distDocSet = new GeoDistanceRangeDocSet(context.reader().maxDoc(), acceptedDocs, values, fixedSourceDistance, distanceBoundingCheck, inclusiveLowerPoint, inclusiveUpperPoint);
         if (boundingBoxDocSet == null) {
             return distDocSet;
         } else {
@@ -147,7 +140,8 @@ public class GeoDistanceRangeFilter extends Filter {
         if (Double.compare(filter.inclusiveUpperPoint, inclusiveUpperPoint) != 0) return false;
         if (Double.compare(filter.lat, lat) != 0) return false;
         if (Double.compare(filter.lon, lon) != 0) return false;
-        if (fieldName != null ? !fieldName.equals(filter.fieldName) : filter.fieldName != null) return false;
+        if (!indexFieldData.getFieldNames().indexName().equals(filter.indexFieldData.getFieldNames().indexName()))
+            return false;
         if (geoDistance != filter.geoDistance) return false;
 
         return true;
@@ -155,7 +149,7 @@ public class GeoDistanceRangeFilter extends Filter {
 
     @Override
     public String toString() {
-        return "GeoDistanceRangeFilter(" + fieldName + ", " + geoDistance + ", [" + inclusiveLowerPoint + " - " + inclusiveUpperPoint + "], " + lat + ", " + lon + ")";
+        return "GeoDistanceRangeFilter(" + indexFieldData.getFieldNames().indexName() + ", " + geoDistance + ", [" + inclusiveLowerPoint + " - " + inclusiveUpperPoint + "], " + lat + ", " + lon + ")";
     }
 
     @Override
@@ -171,22 +165,22 @@ public class GeoDistanceRangeFilter extends Filter {
         temp = inclusiveUpperPoint != +0.0d ? Double.doubleToLongBits(inclusiveUpperPoint) : 0L;
         result = 31 * result + (int) (temp ^ (temp >>> 32));
         result = 31 * result + (geoDistance != null ? geoDistance.hashCode() : 0);
-        result = 31 * result + (fieldName != null ? fieldName.hashCode() : 0);
+        result = 31 * result + indexFieldData.getFieldNames().indexName().hashCode();
         return result;
     }
 
     public static class GeoDistanceRangeDocSet extends MatchDocIdSet {
 
-        private final GeoPointFieldData fieldData;
+        private final GeoPointValues values;
         private final GeoDistance.FixedSourceDistance fixedSourceDistance;
         private final GeoDistance.DistanceBoundingCheck distanceBoundingCheck;
         private final double inclusiveLowerPoint; // in miles
         private final double inclusiveUpperPoint; // in miles
 
-        public GeoDistanceRangeDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointFieldData fieldData, GeoDistance.FixedSourceDistance fixedSourceDistance, GeoDistance.DistanceBoundingCheck distanceBoundingCheck,
+        public GeoDistanceRangeDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointValues values, GeoDistance.FixedSourceDistance fixedSourceDistance, GeoDistance.DistanceBoundingCheck distanceBoundingCheck,
                                       double inclusiveLowerPoint, double inclusiveUpperPoint) {
             super(maxDoc, acceptDocs);
-            this.fieldData = fieldData;
+            this.values = values;
             this.fixedSourceDistance = fixedSourceDistance;
             this.distanceBoundingCheck = distanceBoundingCheck;
             this.inclusiveLowerPoint = inclusiveLowerPoint;
@@ -200,18 +194,16 @@ public class GeoDistanceRangeFilter extends Filter {
 
         @Override
         protected boolean matchDoc(int doc) {
-            if (!fieldData.hasValue(doc)) {
+            if (!values.hasValue(doc)) {
                 return false;
             }
 
-            if (fieldData.multiValued()) {
-                double[] lats = fieldData.latValues(doc);
-                double[] lons = fieldData.lonValues(doc);
-                for (int i = 0; i < lats.length; i++) {
-                    double lat = lats[i];
-                    double lon = lons[i];
-                    if (distanceBoundingCheck.isWithin(lat, lon)) {
-                        double d = fixedSourceDistance.calculate(lat, lon);
+            if (values.isMultiValued()) {
+                GeoPointValues.Iter iter = values.getIter(doc);
+                while (iter.hasNext()) {
+                    GeoPoint point = iter.next();
+                    if (distanceBoundingCheck.isWithin(point.lat(), point.lon())) {
+                        double d = fixedSourceDistance.calculate(point.lat(), point.lon());
                         if (d >= inclusiveLowerPoint && d <= inclusiveUpperPoint) {
                             return true;
                         }
@@ -219,10 +211,9 @@ public class GeoDistanceRangeFilter extends Filter {
                 }
                 return false;
             } else {
-                double lat = fieldData.latValue(doc);
-                double lon = fieldData.lonValue(doc);
-                if (distanceBoundingCheck.isWithin(lat, lon)) {
-                    double d = fixedSourceDistance.calculate(lat, lon);
+                GeoPoint point = values.getValue(doc);
+                if (distanceBoundingCheck.isWithin(point.lat(), point.lon())) {
+                    double d = fixedSourceDistance.calculate(point.lat(), point.lon());
                     if (d >= inclusiveLowerPoint && d <= inclusiveUpperPoint) {
                         return true;
                     }
diff --git a/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonFilter.java b/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonFilter.java
index 06303de1c33..bb7e5b9a3df 100644
--- a/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonFilter.java
+++ b/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonFilter.java
@@ -25,9 +25,9 @@ import org.apache.lucene.search.Filter;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.lucene.docset.MatchDocIdSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -39,14 +39,11 @@ public class GeoPolygonFilter extends Filter {
 
     private final Point[] points;
 
-    private final String fieldName;
+    private final IndexGeoPointFieldData indexFieldData;
 
-    private final FieldDataCache fieldDataCache;
-
-    public GeoPolygonFilter(Point[] points, String fieldName, FieldDataCache fieldDataCache) {
+    public GeoPolygonFilter(Point[] points, IndexGeoPointFieldData indexFieldData) {
         this.points = points;
-        this.fieldName = fieldName;
-        this.fieldDataCache = fieldDataCache;
+        this.indexFieldData = indexFieldData;
     }
 
     public Point[] points() {
@@ -54,27 +51,27 @@ public class GeoPolygonFilter extends Filter {
     }
 
     public String fieldName() {
-        return this.fieldName;
+        return indexFieldData.getFieldNames().indexName();
     }
 
     @Override
     public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptedDocs) throws IOException {
-        final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, context.reader(), fieldName);
-        return new GeoPolygonDocIdSet(context.reader().maxDoc(), acceptedDocs, fieldData, points);
+        final GeoPointValues values = indexFieldData.load(context).getGeoPointValues();
+        return new GeoPolygonDocIdSet(context.reader().maxDoc(), acceptedDocs, values, points);
     }
 
     @Override
     public String toString() {
-        return "GeoPolygonFilter(" + fieldName + ", " + Arrays.toString(points) + ")";
+        return "GeoPolygonFilter(" + indexFieldData.getFieldNames().indexName() + ", " + Arrays.toString(points) + ")";
     }
 
     public static class GeoPolygonDocIdSet extends MatchDocIdSet {
-        private final GeoPointFieldData fieldData;
+        private final GeoPointValues values;
         private final Point[] points;
 
-        public GeoPolygonDocIdSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointFieldData fieldData, Point[] points) {
+        public GeoPolygonDocIdSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointValues values, Point[] points) {
             super(maxDoc, acceptDocs);
-            this.fieldData = fieldData;
+            this.values = values;
             this.points = points;
         }
 
@@ -85,22 +82,21 @@ public class GeoPolygonFilter extends Filter {
 
         @Override
         protected boolean matchDoc(int doc) {
-            if (!fieldData.hasValue(doc)) {
+            if (!values.hasValue(doc)) {
                 return false;
             }
 
-            if (fieldData.multiValued()) {
-                double[] lats = fieldData.latValues(doc);
-                double[] lons = fieldData.lonValues(doc);
-                for (int i = 0; i < lats.length; i++) {
-                    if (pointInPolygon(points, lats[i], lons[i])) {
+            if (values.isMultiValued()) {
+                GeoPointValues.Iter iter = values.getIter(doc);
+                while (iter.hasNext()) {
+                    GeoPoint point = iter.next();
+                    if (pointInPolygon(points, point.lat(), point.lon())) {
                         return true;
                     }
                 }
             } else {
-                double lat = fieldData.latValue(doc);
-                double lon = fieldData.lonValue(doc);
-                return pointInPolygon(points, lat, lon);
+                GeoPoint point = values.getValue(doc);
+                return pointInPolygon(points, point.lat(), point.lon());
             }
             return false;
         }
diff --git a/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxFilter.java b/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxFilter.java
index 29ce5e4a7fb..f97b98c5a13 100644
--- a/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxFilter.java
+++ b/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxFilter.java
@@ -25,9 +25,9 @@ import org.apache.lucene.search.Filter;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.lucene.docset.MatchDocIdSet;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.mapper.geo.GeoPoint;
 
 import java.io.IOException;
 
@@ -37,18 +37,14 @@ import java.io.IOException;
 public class InMemoryGeoBoundingBoxFilter extends Filter {
 
     private final Point topLeft;
-
     private final Point bottomRight;
 
-    private final String fieldName;
+    private final IndexGeoPointFieldData indexFieldData;
 
-    private final FieldDataCache fieldDataCache;
-
-    public InMemoryGeoBoundingBoxFilter(Point topLeft, Point bottomRight, String fieldName, FieldDataCache fieldDataCache) {
+    public InMemoryGeoBoundingBoxFilter(Point topLeft, Point bottomRight, IndexGeoPointFieldData indexFieldData) {
         this.topLeft = topLeft;
         this.bottomRight = bottomRight;
-        this.fieldName = fieldName;
-        this.fieldDataCache = fieldDataCache;
+        this.indexFieldData = indexFieldData;
     }
 
     public Point topLeft() {
@@ -60,34 +56,34 @@ public class InMemoryGeoBoundingBoxFilter extends Filter {
     }
 
     public String fieldName() {
-        return fieldName;
+        return indexFieldData.getFieldNames().indexName();
     }
 
     @Override
     public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptedDocs) throws IOException {
-        final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, context.reader(), fieldName);
+        final GeoPointValues values = indexFieldData.load(context).getGeoPointValues();
 
         //checks to see if bounding box crosses 180 degrees
         if (topLeft.lon > bottomRight.lon) {
-            return new Meridian180GeoBoundingBoxDocSet(context.reader().maxDoc(), acceptedDocs, fieldData, topLeft, bottomRight);
+            return new Meridian180GeoBoundingBoxDocSet(context.reader().maxDoc(), acceptedDocs, values, topLeft, bottomRight);
         } else {
-            return new GeoBoundingBoxDocSet(context.reader().maxDoc(), acceptedDocs, fieldData, topLeft, bottomRight);
+            return new GeoBoundingBoxDocSet(context.reader().maxDoc(), acceptedDocs, values, topLeft, bottomRight);
         }
     }
 
     @Override
     public String toString() {
-        return "GeoBoundingBoxFilter(" + fieldName + ", " + topLeft + ", " + bottomRight + ")";
+        return "GeoBoundingBoxFilter(" + indexFieldData.getFieldNames().indexName() + ", " + topLeft + ", " + bottomRight + ")";
     }
 
     public static class Meridian180GeoBoundingBoxDocSet extends MatchDocIdSet {
-        private final GeoPointFieldData fieldData;
+        private final GeoPointValues values;
         private final Point topLeft;
         private final Point bottomRight;
 
-        public Meridian180GeoBoundingBoxDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointFieldData fieldData, Point topLeft, Point bottomRight) {
+        public Meridian180GeoBoundingBoxDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointValues values, Point topLeft, Point bottomRight) {
             super(maxDoc, acceptDocs);
-            this.fieldData = fieldData;
+            this.values = values;
             this.topLeft = topLeft;
             this.bottomRight = bottomRight;
         }
@@ -99,27 +95,24 @@ public class InMemoryGeoBoundingBoxFilter extends Filter {
 
         @Override
         protected boolean matchDoc(int doc) {
-            if (!fieldData.hasValue(doc)) {
+            if (!values.hasValue(doc)) {
                 return false;
             }
 
-            if (fieldData.multiValued()) {
-                double[] lats = fieldData.latValues(doc);
-                double[] lons = fieldData.lonValues(doc);
-                for (int i = 0; i < lats.length; i++) {
-                    double lat = lats[i];
-                    double lon = lons[i];
-                    if (((topLeft.lon <= lon || bottomRight.lon >= lon)) &&
-                            (topLeft.lat >= lat && bottomRight.lat <= lat)) {
+            if (values.isMultiValued()) {
+                GeoPointValues.Iter iter = values.getIter(doc);
+                while (iter.hasNext()) {
+                    GeoPoint point = iter.next();
+                    if (((topLeft.lon <= point.lon() || bottomRight.lon >= point.lon())) &&
+                            (topLeft.lat >= point.lat() && bottomRight.lat <= point.lat())) {
                         return true;
                     }
                 }
             } else {
-                double lat = fieldData.latValue(doc);
-                double lon = fieldData.lonValue(doc);
+                GeoPoint point = values.getValue(doc);
 
-                if (((topLeft.lon <= lon || bottomRight.lon >= lon)) &&
-                        (topLeft.lat >= lat && bottomRight.lat <= lat)) {
+                if (((topLeft.lon <= point.lon() || bottomRight.lon >= point.lon())) &&
+                        (topLeft.lat >= point.lat() && bottomRight.lat <= point.lat())) {
                     return true;
                 }
             }
@@ -128,13 +121,13 @@ public class InMemoryGeoBoundingBoxFilter extends Filter {
     }
 
     public static class GeoBoundingBoxDocSet extends MatchDocIdSet {
-        private final GeoPointFieldData fieldData;
+        private final GeoPointValues values;
         private final Point topLeft;
         private final Point bottomRight;
 
-        public GeoBoundingBoxDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointFieldData fieldData, Point topLeft, Point bottomRight) {
+        public GeoBoundingBoxDocSet(int maxDoc, @Nullable Bits acceptDocs, GeoPointValues values, Point topLeft, Point bottomRight) {
             super(maxDoc, acceptDocs);
-            this.fieldData = fieldData;
+            this.values = values;
             this.topLeft = topLeft;
             this.bottomRight = bottomRight;
         }
@@ -146,25 +139,23 @@ public class InMemoryGeoBoundingBoxFilter extends Filter {
 
         @Override
         protected boolean matchDoc(int doc) {
-            if (!fieldData.hasValue(doc)) {
+            if (!values.hasValue(doc)) {
                 return false;
             }
 
-            if (fieldData.multiValued()) {
-                double[] lats = fieldData.latValues(doc);
-                double[] lons = fieldData.lonValues(doc);
-                for (int i = 0; i < lats.length; i++) {
-                    if (topLeft.lon <= lons[i] && bottomRight.lon >= lons[i]
-                            && topLeft.lat >= lats[i] && bottomRight.lat <= lats[i]) {
+            if (values.isMultiValued()) {
+                GeoPointValues.Iter iter = values.getIter(doc);
+                while (iter.hasNext()) {
+                    GeoPoint point = iter.next();
+                    if (topLeft.lon <= point.lon() && bottomRight.lon >= point.lon()
+                            && topLeft.lat >= point.lat() && bottomRight.lat <= point.lat()) {
                         return true;
                     }
                 }
             } else {
-                double lat = fieldData.latValue(doc);
-                double lon = fieldData.lonValue(doc);
-
-                if (topLeft.lon <= lon && bottomRight.lon >= lon
-                        && topLeft.lat >= lat && bottomRight.lat <= lat) {
+                GeoPoint point = values.getValue(doc);
+                if (topLeft.lon <= point.lon() && bottomRight.lon >= point.lon()
+                        && topLeft.lat >= point.lat() && bottomRight.lat <= point.lat()) {
                     return true;
                 }
             }
diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 6284315d454..e535b8ede33 100644
--- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -34,7 +34,6 @@ import org.elasticsearch.common.lucene.search.XFilteredQuery;
 import org.elasticsearch.common.lucene.search.function.BoostScoreFunction;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.cache.filter.FilterCache;
 import org.elasticsearch.index.cache.id.IdCache;
 import org.elasticsearch.index.engine.Engine;
@@ -356,10 +355,6 @@ public class SearchContext implements Releasable {
         return indexService.cache().filter();
     }
 
-    public FieldDataCache fieldDataCache() {
-        return indexService.cache().fieldData();
-    }
-
     public IndexFieldDataService fieldData() {
         return indexService.fieldData();
     }

From af757fd8217cc42ab49300426c0b51795135f0ac Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 14:57:26 +0100
Subject: [PATCH 30/35] more usage of field data note, removed field data from
 cache stats, it will have its own stats later on (cache part is really
 misleading)

---
 .../TransportClearIndicesCacheAction.java     |  6 +--
 .../elasticsearch/index/cache/CacheStats.java | 42 +------------------
 .../elasticsearch/index/cache/IndexCache.java | 18 ++------
 .../index/percolator/PercolatorExecutor.java  | 19 +++++----
 4 files changed, 17 insertions(+), 68 deletions(-)

diff --git a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
index ea40b7fd4dd..fd937a89cd6 100644
--- a/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
+++ b/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
@@ -127,10 +127,10 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastOperatio
             if (request.fieldDataCache()) {
                 clearedAtLeastOne = true;
                 if (request.fields() == null || request.fields().length == 0) {
-                    service.cache().fieldData().clear("api");
+                    service.fieldData().clear();
                 } else {
                     for (String field : request.fields()) {
-                        service.cache().fieldData().clear("api", field);
+                        service.fieldData().clearField(field);
                     }
                 }
             }
@@ -142,7 +142,7 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastOperatio
                 if (request.fields() != null && request.fields().length > 0) {
                     // only clear caches relating to the specified fields
                     for (String field : request.fields()) {
-                        service.cache().fieldData().clear("api", field);
+                        service.fieldData().clearField(field);
                     }
                 } else {
                     service.cache().clear("api");
diff --git a/src/main/java/org/elasticsearch/index/cache/CacheStats.java b/src/main/java/org/elasticsearch/index/cache/CacheStats.java
index eb1c420ae4b..d8ce6149da1 100644
--- a/src/main/java/org/elasticsearch/index/cache/CacheStats.java
+++ b/src/main/java/org/elasticsearch/index/cache/CacheStats.java
@@ -34,42 +34,28 @@ import java.io.IOException;
  */
 public class CacheStats implements Streamable, ToXContent {
 
-    long fieldEvictions;
     long filterEvictions;
     long filterCount;
-    long fieldSize;
     long filterSize;
     long idCacheSize;
 
     public CacheStats() {
     }
 
-    public CacheStats(long fieldEvictions, long filterEvictions, long fieldSize, long filterSize, long filterCount, long idCacheSize) {
-        this.fieldEvictions = fieldEvictions;
+    public CacheStats(long filterEvictions, long filterSize, long filterCount, long idCacheSize) {
         this.filterEvictions = filterEvictions;
-        this.fieldSize = fieldSize;
         this.filterSize = filterSize;
         this.filterCount = filterCount;
         this.idCacheSize = idCacheSize;
     }
 
     public void add(CacheStats stats) {
-        this.fieldEvictions += stats.fieldEvictions;
         this.filterEvictions += stats.filterEvictions;
-        this.fieldSize += stats.fieldSize;
         this.filterSize += stats.filterSize;
         this.filterCount += stats.filterCount;
         this.idCacheSize += stats.idCacheSize;
     }
 
-    public long fieldEvictions() {
-        return this.fieldEvictions;
-    }
-
-    public long getFieldEvictions() {
-        return this.fieldEvictions();
-    }
-
     public long filterEvictions() {
         return this.filterEvictions;
     }
@@ -94,22 +80,6 @@ public class CacheStats implements Streamable, ToXContent {
         return filterCount;
     }
 
-    public long fieldSizeInBytes() {
-        return this.fieldSize;
-    }
-
-    public long getFieldSizeInBytes() {
-        return fieldSizeInBytes();
-    }
-
-    public ByteSizeValue fieldSize() {
-        return new ByteSizeValue(fieldSize);
-    }
-
-    public ByteSizeValue getFieldSize() {
-        return this.fieldSize();
-    }
-
     public long filterSizeInBytes() {
         return this.filterSize;
     }
@@ -145,9 +115,6 @@ public class CacheStats implements Streamable, ToXContent {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.CACHE);
-        builder.field(Fields.FIELD_EVICTIONS, fieldEvictions);
-        builder.field(Fields.FIELD_SIZE, fieldSize().toString());
-        builder.field(Fields.FIELD_SIZE_IN_BYTES, fieldSize);
         builder.field(Fields.FILTER_COUNT, filterCount);
         builder.field(Fields.FILTER_EVICTIONS, filterEvictions);
         builder.field(Fields.FILTER_SIZE, filterSize().toString());
@@ -160,9 +127,6 @@ public class CacheStats implements Streamable, ToXContent {
 
     static final class Fields {
         static final XContentBuilderString CACHE = new XContentBuilderString("cache");
-        static final XContentBuilderString FIELD_SIZE = new XContentBuilderString("field_size");
-        static final XContentBuilderString FIELD_SIZE_IN_BYTES = new XContentBuilderString("field_size_in_bytes");
-        static final XContentBuilderString FIELD_EVICTIONS = new XContentBuilderString("field_evictions");
         static final XContentBuilderString FILTER_EVICTIONS = new XContentBuilderString("filter_evictions");
         static final XContentBuilderString FILTER_COUNT = new XContentBuilderString("filter_count");
         static final XContentBuilderString FILTER_SIZE = new XContentBuilderString("filter_size");
@@ -179,9 +143,7 @@ public class CacheStats implements Streamable, ToXContent {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        fieldEvictions = in.readVLong();
         filterEvictions = in.readVLong();
-        fieldSize = in.readVLong();
         filterSize = in.readVLong();
         filterCount = in.readVLong();
         idCacheSize = in.readVLong();
@@ -189,9 +151,7 @@ public class CacheStats implements Streamable, ToXContent {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(fieldEvictions);
         out.writeVLong(filterEvictions);
-        out.writeVLong(fieldSize);
         out.writeVLong(filterSize);
         out.writeVLong(filterCount);
         out.writeVLong(idCacheSize);
diff --git a/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/src/main/java/org/elasticsearch/index/cache/IndexCache.java
index 0e2957bdded..7b184c321eb 100644
--- a/src/main/java/org/elasticsearch/index/cache/IndexCache.java
+++ b/src/main/java/org/elasticsearch/index/cache/IndexCache.java
@@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
 import org.elasticsearch.index.cache.filter.FilterCache;
 import org.elasticsearch.index.cache.id.IdCache;
 import org.elasticsearch.index.cache.query.parser.QueryParserCache;
@@ -44,8 +43,6 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
 
     private final FilterCache filterCache;
 
-    private final FieldDataCache fieldDataCache;
-
     private final QueryParserCache queryParserCache;
 
     private final IdCache idCache;
@@ -58,11 +55,9 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
     private CacheStats latestCacheStats;
 
     @Inject
-    public IndexCache(Index index, @IndexSettings Settings indexSettings, FilterCache filterCache, FieldDataCache fieldDataCache,
-                      QueryParserCache queryParserCache, IdCache idCache) {
+    public IndexCache(Index index, @IndexSettings Settings indexSettings, FilterCache filterCache, QueryParserCache queryParserCache, IdCache idCache) {
         super(index, indexSettings);
         this.filterCache = filterCache;
-        this.fieldDataCache = fieldDataCache;
         this.queryParserCache = queryParserCache;
         this.idCache = idCache;
 
@@ -81,7 +76,7 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
 
     public synchronized void invalidateCache() {
         FilterCache.EntriesStats filterEntriesStats = filterCache.entriesStats();
-        latestCacheStats = new CacheStats(fieldDataCache.evictions(), filterCache.evictions(), fieldDataCache.sizeInBytes(), filterEntriesStats.sizeInBytes, filterEntriesStats.count, idCache.sizeInBytes());
+        latestCacheStats = new CacheStats(filterCache.evictions(), filterEntriesStats.sizeInBytes, filterEntriesStats.count, idCache.sizeInBytes());
         latestCacheStatsTimestamp = System.currentTimeMillis();
     }
 
@@ -89,7 +84,7 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
         long timestamp = System.currentTimeMillis();
         if ((timestamp - latestCacheStatsTimestamp) > refreshInterval.millis()) {
             FilterCache.EntriesStats filterEntriesStats = filterCache.entriesStats();
-            latestCacheStats = new CacheStats(fieldDataCache.evictions(), filterCache.evictions(), fieldDataCache.sizeInBytes(), filterEntriesStats.sizeInBytes, filterEntriesStats.count, idCache.sizeInBytes());
+            latestCacheStats = new CacheStats(filterCache.evictions(), filterEntriesStats.sizeInBytes, filterEntriesStats.count, idCache.sizeInBytes());
             latestCacheStatsTimestamp = timestamp;
         }
         return latestCacheStats;
@@ -99,10 +94,6 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
         return filterCache;
     }
 
-    public FieldDataCache fieldData() {
-        return fieldDataCache;
-    }
-
     public IdCache idCache() {
         return this.idCache;
     }
@@ -114,7 +105,6 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
     @Override
     public void close() throws ElasticSearchException {
         filterCache.close();
-        fieldDataCache.close();
         idCache.close();
         queryParserCache.close();
         if (clusterService != null) {
@@ -124,13 +114,11 @@ public class IndexCache extends AbstractIndexComponent implements CloseableCompo
 
     public void clear(IndexReader reader) {
         filterCache.clear(reader);
-        fieldDataCache.clear(reader);
         idCache.clear(reader);
     }
 
     public void clear(String reason) {
         filterCache.clear(reason);
-        fieldDataCache.clear(reason);
         idCache.clear();
         queryParserCache.clear();
     }
diff --git a/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java b/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java
index 54cd58330e9..187a41d1e45 100644
--- a/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java
+++ b/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java
@@ -46,12 +46,10 @@ import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.fielddata.BytesValues;
+import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.query.IndexQueryParserService;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -363,6 +361,7 @@ public class PercolatorExecutor extends AbstractIndexComponent {
     }
 
     static class QueryCollector extends Collector {
+        private final IndexFieldData uidFieldData;
         private final IndexSearcher searcher;
         private final IndexService percolatorIndex;
         private final List<String> matches;
@@ -371,7 +370,7 @@ public class PercolatorExecutor extends AbstractIndexComponent {
 
         private final Lucene.ExistsCollector collector = new Lucene.ExistsCollector();
 
-        private FieldData fieldData;
+        private BytesValues values;
 
         QueryCollector(ESLogger logger, Map<String, Query> queries, IndexSearcher searcher, IndexService percolatorIndex, List<String> matches) {
             this.logger = logger;
@@ -379,6 +378,8 @@ public class PercolatorExecutor extends AbstractIndexComponent {
             this.searcher = searcher;
             this.percolatorIndex = percolatorIndex;
             this.matches = matches;
+            // TODO: when we move to a UID level mapping def on the index level, we can use that one, now, its per type, and we can't easily choose one
+            this.uidFieldData = percolatorIndex.fieldData().getForField(new FieldMapper.Names(UidFieldMapper.NAME), new FieldDataType("string", "paged_bytes"));
         }
 
         @Override
@@ -387,7 +388,7 @@ public class PercolatorExecutor extends AbstractIndexComponent {
 
         @Override
         public void collect(int doc) throws IOException {
-            BytesRef uid = fieldData.stringValue(doc);
+            BytesRef uid = values.getValue(doc);
             if (uid == null) {
                 return;
             }
@@ -412,7 +413,7 @@ public class PercolatorExecutor extends AbstractIndexComponent {
         @Override
         public void setNextReader(AtomicReaderContext context) throws IOException {
             // we use the UID because id might not be indexed
-            fieldData = percolatorIndex.cache().fieldData().cache(FieldDataType.DefaultTypes.STRING, context.reader(), UidFieldMapper.NAME);
+            values = uidFieldData.load(context).getBytesValues();
         }
 
         @Override

From c67386f6448ff37fa9cb85c31b6cfc706161c268 Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 15:20:15 +0100
Subject: [PATCH 31/35] properly invalidate on core closed reader

---
 .../index/fielddata/FieldDataStats.java       | 82 +++++++++++++++++++
 .../index/fielddata/IndexFieldDataCache.java  |  3 +
 .../fielddata/IndexFieldDataService.java      |  5 ++
 3 files changed, 90 insertions(+)
 create mode 100644 src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java

diff --git a/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java b/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java
new file mode 100644
index 00000000000..2ef0b3ce7b0
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+
+import java.io.IOException;
+
+/**
+ */
+public class FieldDataStats implements Streamable, ToXContent {
+
+    long memorySize;
+
+    public FieldDataStats() {
+
+    }
+
+    public FieldDataStats(long memorySize) {
+        this.memorySize = memorySize;
+    }
+
+    public void add(FieldDataStats stats) {
+        this.memorySize += stats.memorySize;
+    }
+
+    public long getMemorySizeInBytes() {
+        return this.memorySize;
+    }
+
+    public ByteSizeValue getMemorySize() {
+        return new ByteSizeValue(memorySize);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        memorySize = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(memorySize);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.FIELD_DATA);
+        builder.field(Fields.MEMORY_SIZE, memorySize);
+        builder.field(Fields.MEMORY_SIZE_IN_BYTES, getMemorySize().toString());
+        builder.endObject();
+        return builder;
+    }
+
+    static final class Fields {
+        static final XContentBuilderString FIELD_DATA = new XContentBuilderString("field_data");
+        static final XContentBuilderString MEMORY_SIZE = new XContentBuilderString("memory_size");
+        static final XContentBuilderString MEMORY_SIZE_IN_BYTES = new XContentBuilderString("memory_size_in_bytes");
+    }
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
index 3feb5a19f87..57d7643170d 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
@@ -59,6 +59,9 @@ public interface IndexFieldDataCache {
             return (FD) cache.get(context.reader().getCoreCacheKey(), new Callable<AtomicFieldData>() {
                 @Override
                 public AtomicFieldData call() throws Exception {
+                    if (context.reader() instanceof SegmentReader) {
+                        ((SegmentReader) context.reader()).addCoreClosedListener(FieldBased.this);
+                    }
                     return indexFieldData.loadDirect(context);
                 }
             });
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index b97a0f3ca02..a60db479ec3 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -96,6 +96,11 @@ public class IndexFieldDataService extends AbstractIndexComponent {
         }
     }
 
+    public FieldDataStats stats() {
+        // TODO: compute the memory size here...
+        return new FieldDataStats();
+    }
+
     public <IFD extends IndexFieldData> IFD getForField(FieldMapper mapper) {
         return getForField(mapper.names(), mapper.fieldDataType2());
     }

From 6b92b592b4f4256f57d4ea424cebac1f46e39bfa Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 15:30:05 +0100
Subject: [PATCH 32/35] allow to clear by reader the new field data cache

---
 .../index/fielddata/AbstractIndexFieldData.java           | 6 ++++++
 .../org/elasticsearch/index/fielddata/IndexFieldData.java | 3 +++
 .../index/fielddata/IndexFieldDataCache.java              | 8 ++++++++
 .../index/fielddata/IndexFieldDataService.java            | 7 +++++++
 .../index/percolator/PercolatorExecutor.java              | 6 +++++-
 5 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
index d6f6824809f..54751f306be 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
@@ -1,5 +1,6 @@
 package org.elasticsearch.index.fielddata;
 
+import org.apache.lucene.index.IndexReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
@@ -30,4 +31,9 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
     public void clear() {
         cache.clear(index, fieldNames.indexName());
     }
+
+    @Override
+    public void clear(IndexReader reader) {
+        cache.clear(index, reader);
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index 1b74ef9319a..8cd29aeca4c 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.FieldComparatorSource;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.Nullable;
@@ -63,6 +64,8 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
      */
     void clear();
 
+    void clear(IndexReader reader);
+
     // we need this extended source we we have custom comparators to reuse our field data
     // in this case, we need to reduce type that will be used when search results are reduced
     // on another node (we don't have the custom source them...)
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
index 57d7643170d..014943f46a6 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.fielddata;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.SegmentReader;
 import org.elasticsearch.index.Index;
 
@@ -38,6 +39,8 @@ public interface IndexFieldDataCache {
 
     void clear(Index index, String fieldName);
 
+    void clear(Index index, IndexReader reader);
+
     /**
      * The resident field data cache is a *per field* cache that keeps all the values in memory.
      */
@@ -76,6 +79,11 @@ public interface IndexFieldDataCache {
         public void clear(Index index, String fieldName) {
             cache.invalidateAll();
         }
+
+        @Override
+        public void clear(Index index, IndexReader reader) {
+            cache.invalidate(reader.getCoreCacheKey());
+        }
     }
 
     static class Resident extends FieldBased {
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index a60db479ec3..ce8b5fbd61b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.fielddata;
 
 import com.google.common.collect.ImmutableMap;
+import org.apache.lucene.index.IndexReader;
 import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.collect.Tuple;
@@ -96,6 +97,12 @@ public class IndexFieldDataService extends AbstractIndexComponent {
         }
     }
 
+    public void clear(IndexReader reader) {
+        for (IndexFieldData indexFieldData : loadedFieldData.values()) {
+            indexFieldData.clear(reader);
+        }
+    }
+
     public FieldDataStats stats() {
         // TODO: compute the memory size here...
         return new FieldDataStats();
diff --git a/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java b/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java
index 187a41d1e45..4e030cbb18c 100644
--- a/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java
+++ b/src/main/java/org/elasticsearch/index/percolator/PercolatorExecutor.java
@@ -49,6 +49,7 @@ import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.BytesValues;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.query.IndexQueryParserService;
@@ -153,6 +154,7 @@ public class PercolatorExecutor extends AbstractIndexComponent {
     private final IndexQueryParserService queryParserService;
 
     private final IndexCache indexCache;
+    private final IndexFieldDataService fieldDataService;
 
     private final Map<String, Query> queries = ConcurrentCollections.newConcurrentMap();
 
@@ -162,11 +164,12 @@ public class PercolatorExecutor extends AbstractIndexComponent {
     @Inject
     public PercolatorExecutor(Index index, @IndexSettings Settings indexSettings,
                               MapperService mapperService, IndexQueryParserService queryParserService,
-                              IndexCache indexCache) {
+                              IndexCache indexCache, IndexFieldDataService fieldDataService) {
         super(index, indexSettings);
         this.mapperService = mapperService;
         this.queryParserService = queryParserService;
         this.indexCache = indexCache;
+        this.fieldDataService = fieldDataService;
     }
 
     public void setIndicesService(IndicesService indicesService) {
@@ -347,6 +350,7 @@ public class PercolatorExecutor extends AbstractIndexComponent {
         } finally {
             // explicitly clear the reader, since we can only register on callback on SegmentReader
             indexCache.clear(searcher.getIndexReader());
+            fieldDataService.clear(searcher.getIndexReader());
         }
 
         return new Response(matches, request.doc().mappingsModified());

From 09cc70b8c91c25ebf50f592fd9f8f52ac4e67028 Mon Sep 17 00:00:00 2001
From: uboness <uboness@gmail.com>
Date: Tue, 22 Jan 2013 14:45:09 +0100
Subject: [PATCH 33/35] added predefined empty implementation for all atomic
 field datas

---
 .../index/fielddata/AtomicFieldData.java      |   1 +
 .../fielddata/AtomicGeoPointFieldData.java    |   3 +-
 .../fielddata/AtomicNumericFieldData.java     |   1 +
 .../fielddata/AtomicOrdinalFieldData.java     |   1 +
 .../index/fielddata/ByteValues.java           |  39 +++
 .../index/fielddata/BytesValues.java          |  47 ++++
 .../index/fielddata/DoubleValues.java         |  39 +++
 .../index/fielddata/FloatValues.java          |  39 +++
 .../index/fielddata/GeoPointValues.java       |  56 +++-
 .../index/fielddata/HashedBytesValues.java    |  36 +++
 .../index/fielddata/IntValues.java            |  39 +++
 .../index/fielddata/LongValues.java           |  39 +++
 .../index/fielddata/OrdinalsBytesValues.java  |  31 +++
 .../fielddata/OrdinalsHashedBytesValues.java  |  25 ++
 .../index/fielddata/OrdinalsStringValues.java |  20 ++
 .../index/fielddata/ScriptDocValues.java      |  14 +
 .../index/fielddata/ShortValues.java          |  39 +++
 .../index/fielddata/StringValues.java         |  34 +++
 .../index/fielddata/ordinals/Ordinals.java    |   3 +
 .../plain/ByteArrayAtomicFieldData.java       |  74 ++++++
 .../plain/ByteArrayIndexFieldData.java        |   2 +-
 .../ConcreteBytesRefAtomicFieldData.java      |  54 +++-
 .../plain/ConcreteBytesRefIndexFieldData.java |   3 +-
 .../plain/DoubleArrayAtomicFieldData.java     |  74 ++++++
 .../plain/DoubleArrayIndexFieldData.java      |   2 +-
 .../plain/FloatArrayAtomicFieldData.java      |  74 ++++++
 .../plain/FloatArrayIndexFieldData.java       |   2 +-
 .../GeoPointDoubleArrayAtomicFieldData.java   |  56 +++-
 .../GeoPointDoubleArrayIndexFieldData.java    |   4 +-
 .../plain/IntArrayAtomicFieldData.java        |  75 ++++++
 .../plain/IntArrayIndexFieldData.java         |   2 +-
 .../plain/LongArrayAtomicFieldData.java       |  74 ++++++
 .../plain/LongArrayIndexFieldData.java        |   2 +-
 .../plain/PagedBytesAtomicFieldData.java      |  59 ++++-
 .../plain/PagedBytesIndexFieldData.java       |   9 +-
 .../plain/ShortArrayAtomicFieldData.java      |  74 ++++++
 .../plain/ShortArrayIndexFieldData.java       |   2 +-
 .../fielddata/NumericFieldDataTests.java      | 242 ++++++++++++++++++
 .../index/fielddata/StringFieldDataTests.java | 121 +++++++++
 39 files changed, 1484 insertions(+), 27 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
index 1648a320397..c2681fab839 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
@@ -63,4 +63,5 @@ public interface AtomicFieldData<Script extends ScriptDocValues> {
      * Returns a "scripting" based values.
      */
     Script getScriptValues();
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
index ddfae5883fd..8996361b797 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -24,4 +24,5 @@ package org.elasticsearch.index.fielddata;
 public interface AtomicGeoPointFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
 
     GeoPointValues getGeoPointValues();
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
index 748c7e07925..3709df8e249 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
@@ -34,4 +34,5 @@ public interface AtomicNumericFieldData<Script extends ScriptDocValues> extends
     FloatValues getFloatValues();
 
     DoubleValues getDoubleValues();
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java
index 7258a8da47c..9aa93b86f69 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java
@@ -37,4 +37,5 @@ public interface AtomicOrdinalFieldData<Script extends ScriptDocValues> extends
      * Use a non thread safe (lightweight) view of the values as strings.
      */
     OrdinalsStringValues getStringValues();
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java b/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
index cebabaeb3bc..8881b3d801b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ByteValues.java
@@ -28,6 +28,8 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
  */
 public interface ByteValues {
 
+    static ByteValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -100,6 +102,43 @@ public interface ByteValues {
         }
     }
 
+    static class Empty implements ByteValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public byte getValue(int docId) {
+            throw new ElasticSearchIllegalStateException("Can't retrieve a value from an empty ByteValues");
+        }
+
+        @Override
+        public byte getValueMissing(int docId, byte missingValue) {
+            return missingValue;
+        }
+
+        @Override
+        public ByteArrayRef getValues(int docId) {
+            return ByteArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
+
     public static class IntBased implements ByteValues {
 
         private final IntValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
index b44faf79d09..bf060b01c13 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
@@ -28,6 +28,8 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
  */
 public interface BytesValues {
 
+    static final BytesValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -122,6 +124,51 @@ public interface BytesValues {
         }
     }
 
+    static class Empty implements BytesValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public BytesRef getValue(int docId) {
+            return null;
+        }
+
+        @Override
+        public BytesRefArrayRef getValues(int docId) {
+            return BytesRefArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+
+        @Override
+        public BytesRef makeSafe(BytesRef bytes) {
+            //todo we can also throw an excepiton here as the only value this method accepts is a scratch value...
+            //todo ...extracted from this ByteValues, in our case, there are not values, so this should never be called!?!?
+            return BytesRef.deepCopyOf(bytes);
+        }
+
+        @Override
+        public BytesRef getValueScratch(int docId, BytesRef ret) {
+            ret.length = 0;
+            return ret;
+        }
+    }
+
     public static class StringBased implements BytesValues {
 
         private final StringValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java b/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
index c7374d4f0a3..64e51329b3c 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java
@@ -28,6 +28,8 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
  */
 public interface DoubleValues {
 
+    static final DoubleValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -100,6 +102,43 @@ public interface DoubleValues {
         }
     }
 
+    static class Empty implements DoubleValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public double getValue(int docId) {
+            throw new ElasticSearchIllegalStateException("Can't retrieve a value from an empty DoubleValues");
+        }
+
+        @Override
+        public double getValueMissing(int docId, double missingValue) {
+            return missingValue;
+        }
+
+        @Override
+        public DoubleArrayRef getValues(int docId) {
+            return DoubleArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
+
     public static class LongBased implements DoubleValues {
 
         private final LongValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/FloatValues.java b/src/main/java/org/elasticsearch/index/fielddata/FloatValues.java
index d03fdbd3341..753a2d9cda5 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/FloatValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/FloatValues.java
@@ -27,6 +27,8 @@ import org.elasticsearch.index.fielddata.util.FloatArrayRef;
  */
 public interface FloatValues {
 
+    static final FloatValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -99,6 +101,43 @@ public interface FloatValues {
         }
     }
 
+    static class Empty implements FloatValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public float getValue(int docId) {
+            throw new ElasticSearchIllegalStateException("Can't retrieve a value from an empty FloatValues");
+        }
+
+        @Override
+        public float getValueMissing(int docId, float missingValue) {
+            return missingValue;
+        }
+
+        @Override
+        public FloatArrayRef getValues(int docId) {
+            return FloatArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
+
     public static class DoubleBased implements FloatValues {
 
         private final DoubleValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java b/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java
index 83707e2a478..a9f7bbe8b8e 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -27,6 +27,8 @@ import org.elasticsearch.index.mapper.geo.GeoPoint;
  */
 public interface GeoPointValues {
 
+    static final GeoPointValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -119,4 +121,56 @@ public interface GeoPointValues {
             }
         }
     }
+
+    static class Empty implements GeoPointValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public GeoPoint getValueSafe(int docId) {
+            return getValue(docId);
+        }
+
+        @Override
+        public Iter getIterSafe(int docId) {
+            return getIter(docId);
+        }
+
+        @Override
+        public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
+
+        }
+
+        @Override
+        public void forEachLatLonValueInDoc(int docId, LatLonValueInDocProc proc) {
+            //To change body of implemented methods use File | Settings | File Templates.
+        }
+
+        @Override
+        public GeoPoint getValue(int docId) {
+            throw new ElasticSearchIllegalStateException("Can't retrieve a value from an empty GeoPointValues");
+        }
+
+        @Override
+        public GeoPointArrayRef getValues(int docId) {
+            return GeoPointArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
index f3479b900f2..1d5baa8a53b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
@@ -27,6 +27,8 @@ import org.elasticsearch.common.lucene.HashedBytesRef;
  */
 public interface HashedBytesValues {
 
+    static final HashedBytesValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -111,6 +113,40 @@ public interface HashedBytesValues {
         }
     }
 
+    static class Empty implements HashedBytesValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public HashedBytesRef getValue(int docId) {
+            return null;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+
+        @Override
+        public HashedBytesRef makeSafe(HashedBytesRef bytes) {
+            //todo maybe better to throw an excepiton here as the only value this method accepts is a scratch value...
+            //todo ...extracted from this ByteValues, in our case, there are not values, so this should never be called!?!?
+            return HashedBytesRef.deepCopyOf(bytes);
+        }
+    }
+
     /**
      * A {@link BytesValues} based implementation.
      */
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IntValues.java b/src/main/java/org/elasticsearch/index/fielddata/IntValues.java
index 1a9f5908dee..e272ebdca51 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IntValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IntValues.java
@@ -27,6 +27,8 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
  */
 public interface IntValues {
 
+    static final IntValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -99,6 +101,43 @@ public interface IntValues {
         }
     }
 
+    static class Empty implements IntValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public int getValue(int docId) {
+            throw new ElasticSearchIllegalStateException("Can't retrieve a value from an empty IntValues");
+        }
+
+        @Override
+        public int getValueMissing(int docId, int missingValue) {
+            return missingValue;
+        }
+
+        @Override
+        public IntArrayRef getValues(int docId) {
+            return IntArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
+
     public static class LongBased implements IntValues {
 
         private final LongValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/LongValues.java b/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
index d275a7329f5..204cc0140bb 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/LongValues.java
@@ -27,6 +27,8 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
  */
 public interface LongValues {
 
+    static final LongValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -100,4 +102,41 @@ public interface LongValues {
         }
     }
 
+    static class Empty implements LongValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public long getValue(int docId) {
+            throw new ElasticSearchIllegalStateException("Can't retrieve a value from an empty LongValues");
+        }
+
+        @Override
+        public long getValueMissing(int docId, long missingValue) {
+            return missingValue;
+        }
+
+        @Override
+        public LongArrayRef getValues(int docId) {
+            return LongArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java
index a51fa893afa..db47a67c4e7 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 
 /**
@@ -39,6 +40,36 @@ public interface OrdinalsBytesValues extends BytesValues {
 
     BytesRef getSafeValueByOrd(int ord);
 
+    public static class Empty extends BytesValues.Empty implements OrdinalsBytesValues {
+
+        private final Ordinals ordinals;
+
+        public Empty(EmptyOrdinals ordinals) {
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return ordinals.ordinals();
+        }
+
+        @Override
+        public BytesRef getValueByOrd(int ord) {
+            return null;
+        }
+
+        @Override
+        public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
+            ret.length = 0;
+            return ret;
+        }
+
+        @Override
+        public BytesRef getSafeValueByOrd(int ord) {
+            return null;
+        }
+    }
+
     public static class StringBased extends BytesValues.StringBased implements OrdinalsBytesValues {
 
         private final OrdinalsStringValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java
index 6f109a9a233..7835d1f3ea7 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.HashedBytesRef;
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 
 /**
@@ -33,6 +34,30 @@ public interface OrdinalsHashedBytesValues extends HashedBytesValues {
 
     HashedBytesRef getSafeValueByOrd(int ord);
 
+    public static class Empty extends HashedBytesValues.Empty implements OrdinalsHashedBytesValues {
+
+        private final Ordinals ordinals;
+
+        public Empty(EmptyOrdinals ordinals) {
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return ordinals.ordinals();
+        }
+
+        @Override
+        public HashedBytesRef getValueByOrd(int ord) {
+            return null;
+        }
+
+        @Override
+        public HashedBytesRef getSafeValueByOrd(int ord) {
+            return null;
+        }
+    }
+
     static class BytesBased extends HashedBytesValues.BytesBased implements OrdinalsHashedBytesValues {
 
         private final OrdinalsBytesValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java
index 9d8c0e1d44a..f6f0e69e91e 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.fielddata;
 
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 
 /**
@@ -28,4 +29,23 @@ public interface OrdinalsStringValues extends StringValues {
     Ordinals.Docs ordinals();
 
     String getValueByOrd(int ord);
+
+    public static class Empty extends StringValues.Empty implements OrdinalsStringValues {
+
+        private final Ordinals ordinals;
+
+        public Empty(EmptyOrdinals ordinals) {
+            this.ordinals = ordinals;
+        }
+
+        @Override
+        public Ordinals.Docs ordinals() {
+            return ordinals.ordinals();
+        }
+
+        @Override
+        public String getValueByOrd(int ord) {
+            return null;
+        }
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index bd187aa5e94..5a88959b721 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -32,10 +32,24 @@ import org.joda.time.MutableDateTime;
  */
 public interface ScriptDocValues {
 
+    static final ScriptDocValues EMPTY = new Empty();
+    static final Strings EMPTY_STRINGS = new Strings(StringValues.EMPTY);
+
     void setNextDocId(int docId);
 
     boolean isEmpty();
 
+    static class Empty implements ScriptDocValues {
+        @Override
+        public void setNextDocId(int docId) {
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return true;
+        }
+    }
+
     static class Strings implements ScriptDocValues {
 
         private final StringValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java b/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
index 0346ff78377..d87bea41773 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ShortValues.java
@@ -28,6 +28,8 @@ import org.elasticsearch.index.fielddata.util.ShortArrayRef;
  */
 public interface ShortValues {
 
+    static final ShortValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -100,6 +102,43 @@ public interface ShortValues {
         }
     }
 
+    static class Empty implements ShortValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public short getValue(int docId) {
+            throw new ElasticSearchIllegalStateException("Can't retrieve a value from an empty ShortValues");
+        }
+
+        @Override
+        public short getValueMissing(int docId, short missingValue) {
+            return missingValue;
+        }
+
+        @Override
+        public ShortArrayRef getValues(int docId) {
+            return ShortArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
+
     public static class IntBased implements ShortValues {
 
         private final IntValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
index e0671bef496..5340282e44b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/StringValues.java
@@ -26,6 +26,8 @@ import org.elasticsearch.index.fielddata.util.*;
  */
 public interface StringValues {
 
+    static final StringValues EMPTY = new Empty();
+
     /**
      * Is one of the documents in this field data values is multi valued?
      */
@@ -100,6 +102,38 @@ public interface StringValues {
         }
     }
 
+    static class Empty implements StringValues {
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean hasValue(int docId) {
+            return false;
+        }
+
+        @Override
+        public String getValue(int docId) {
+            return null;
+        }
+
+        @Override
+        public StringArrayRef getValues(int docId) {
+            return StringArrayRef.EMPTY;
+        }
+
+        @Override
+        public Iter getIter(int docId) {
+            return Iter.Empty.INSTANCE;
+        }
+
+        @Override
+        public void forEachValueInDoc(int docId, ValueInDocProc proc) {
+            proc.onMissing(docId);
+        }
+    }
+
     public static class ByteBased implements StringValues {
 
         private final ByteValues values;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
index b2d25789db7..cdbdffeb93a 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.fielddata.ordinals;
 
+import org.elasticsearch.common.RamUsage;
 import org.elasticsearch.index.fielddata.util.IntArrayRef;
 
 /**
@@ -149,5 +150,7 @@ public interface Ordinals {
                 return actual;
             }
         }
+
     }
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
index a5ef7ee1896..30b6ee25cd4 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
@@ -32,6 +32,8 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
  */
 public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData {
 
+    public static final ByteArrayAtomicFieldData EMPTY = new Empty();
+
     protected final byte[] values;
     private final int numDocs;
 
@@ -47,6 +49,78 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
         return numDocs;
     }
 
+    static class Empty extends ByteArrayAtomicFieldData {
+
+        Empty() {
+            super(null, 0);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return ByteValues.EMPTY;
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return ShortValues.EMPTY;
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return IntValues.EMPTY;
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return LongValues.EMPTY;
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return FloatValues.EMPTY;
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return DoubleValues.EMPTY;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return HashedBytesValues.EMPTY;
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return StringValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+    }
+
     public static class WithOrdinals extends ByteArrayAtomicFieldData {
 
         private final Ordinals ordinals;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
index 3fcb6ab0cdb..4d6b03f21ad 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class ByteArrayIndexFieldData extends AbstractIndexFieldData<ByteArrayAto
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new ByteArrayAtomicFieldData.SingleFixedSet(new byte[1], 0, new FixedBitSet(1));
+            return ByteArrayAtomicFieldData.EMPTY;
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
index fcafcba97bd..5d68dbe558e 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
@@ -23,6 +23,7 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.RamUsage;
 import org.elasticsearch.common.lucene.HashedBytesRef;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
 import org.elasticsearch.index.fielddata.util.IntArrayRef;
@@ -32,9 +33,13 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
  */
 public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
 
+    public static ConcreteBytesRefAtomicFieldData empty(int numDocs) {
+        return new Empty(numDocs);
+    }
+
     // 0 ordinal in values means no value (its null)
     private final BytesRef[] values;
-    private final Ordinals ordinals;
+    protected final Ordinals ordinals;
 
     private int[] hashes;
     private long size = -1;
@@ -586,4 +591,51 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
             }
         }
     }
+
+    static class Empty extends ConcreteBytesRefAtomicFieldData {
+
+        Empty(int numDocs) {
+            super(null, new EmptyOrdinals(numDocs));
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return ordinals.getNumDocs();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public OrdinalsBytesValues getBytesValues() {
+            return new OrdinalsBytesValues.Empty((EmptyOrdinals) ordinals);
+        }
+
+        @Override
+        public OrdinalsHashedBytesValues getHashedBytesValues() {
+            return new OrdinalsHashedBytesValues.Empty((EmptyOrdinals) ordinals);
+        }
+
+        @Override
+        public OrdinalsStringValues getStringValues() {
+            return new OrdinalsStringValues.Empty((EmptyOrdinals) ordinals);
+        }
+
+        @Override
+        public ScriptDocValues.Strings getScriptValues() {
+            return ScriptDocValues.EMPTY_STRINGS;
+        }
+    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
index 30e0520303a..e73a6cd27e3 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -75,7 +74,7 @@ public class ConcreteBytesRefIndexFieldData extends AbstractIndexFieldData<Concr
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new ConcreteBytesRefAtomicFieldData(new BytesRef[1], new EmptyOrdinals(reader.maxDoc()));
+            return ConcreteBytesRefAtomicFieldData.empty(reader.maxDoc());
         }
 
         long size = terms.size();
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
index a04d4841c32..198d87f2605 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
@@ -32,6 +32,8 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
  */
 public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldData {
 
+    public static final DoubleArrayAtomicFieldData EMPTY = new Empty();
+
     protected final double[] values;
     private final int numDocs;
 
@@ -47,6 +49,78 @@ public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldDa
         return numDocs;
     }
 
+    static class Empty extends DoubleArrayAtomicFieldData {
+
+        Empty() {
+            super(null, 0);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return ByteValues.EMPTY;
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return ShortValues.EMPTY;
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return IntValues.EMPTY;
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return LongValues.EMPTY;
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return FloatValues.EMPTY;
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return DoubleValues.EMPTY;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return HashedBytesValues.EMPTY;
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return StringValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+    }
+
     public static class WithOrdinals extends DoubleArrayAtomicFieldData {
 
         private final Ordinals ordinals;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
index d178dffc482..10a679335ba 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new DoubleArrayAtomicFieldData.SingleFixedSet(new double[1], 0, new FixedBitSet(1));
+            return DoubleArrayAtomicFieldData.EMPTY;
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
index 9ae133b7861..8f2a1080146 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
@@ -32,6 +32,8 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
  */
 public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldData {
 
+    public static final FloatArrayAtomicFieldData EMPTY = new Empty();
+
     protected final float[] values;
     private final int numDocs;
 
@@ -47,6 +49,78 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
         return numDocs;
     }
 
+    static class Empty extends FloatArrayAtomicFieldData {
+
+        Empty() {
+            super(null, 0);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return ByteValues.EMPTY;
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return ShortValues.EMPTY;
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return IntValues.EMPTY;
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return LongValues.EMPTY;
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return FloatValues.EMPTY;
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return DoubleValues.EMPTY;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return HashedBytesValues.EMPTY;
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return StringValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+    }
+
     public static class WithOrdinals extends FloatArrayAtomicFieldData {
 
         private final Ordinals ordinals;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
index 3615e560fa1..57d716d6405 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new FloatArrayAtomicFieldData.SingleFixedSet(new float[1], 0, new FixedBitSet(1));
+            return FloatArrayAtomicFieldData.EMPTY;
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
index be12d913ceb..9aa3e104646 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -21,10 +21,7 @@ package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
-import org.elasticsearch.index.fielddata.BytesValues;
-import org.elasticsearch.index.fielddata.HashedBytesValues;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.util.GeoPointArrayRef;
 import org.elasticsearch.index.fielddata.util.IntArrayRef;
@@ -36,6 +33,8 @@ import org.elasticsearch.index.search.geo.GeoHashUtils;
  */
 public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPointFieldData {
 
+    public static final GeoPointDoubleArrayAtomicFieldData EMPTY = new Empty();
+
     protected final double[] lon;
     protected final double[] lat;
     private final int numDocs;
@@ -58,6 +57,53 @@ public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPoi
         return new ScriptDocValues.GeoPoints(getGeoPointValues());
     }
 
+    static class Empty extends GeoPointDoubleArrayAtomicFieldData {
+
+        Empty() {
+            super(null, null, 0);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return HashedBytesValues.EMPTY;
+        }
+
+        @Override
+        public GeoPointValues getGeoPointValues() {
+            return GeoPointValues.EMPTY;
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return StringValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+    }
+
     public static class WithOrdinals extends GeoPointDoubleArrayAtomicFieldData {
 
         private final Ordinals ordinals;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
index c1ccd1aee31..0cef2018aab 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -77,7 +77,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<Ge
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(new double[1], new double[1], 0, new FixedBitSet(1));
+            return GeoPointDoubleArrayAtomicFieldData.EMPTY;
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
index 0bf31844a40..889c5c84cb6 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
@@ -31,6 +31,8 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
  */
 public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData {
 
+    public static final IntArrayAtomicFieldData EMPTY = new Empty();
+
     protected final int[] values;
     private final int numDocs;
 
@@ -46,6 +48,78 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
         return numDocs;
     }
 
+    static class Empty extends IntArrayAtomicFieldData {
+
+        Empty() {
+            super(null, 0);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return ByteValues.EMPTY;
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return ShortValues.EMPTY;
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return IntValues.EMPTY;
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return LongValues.EMPTY;
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return FloatValues.EMPTY;
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return DoubleValues.EMPTY;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return HashedBytesValues.EMPTY;
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return StringValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+    }
+
     public static class WithOrdinals extends IntArrayAtomicFieldData {
 
         private final Ordinals ordinals;
@@ -835,6 +909,7 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
             private final Iter.Single iter = new Iter.Single();
 
             LongValues(int[] values) {
+                assert values.length != 0;
                 this.values = values;
             }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
index 556ab37f75f..58ce91b49d7 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class IntArrayIndexFieldData extends AbstractIndexFieldData<IntArrayAtomi
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new IntArrayAtomicFieldData.SingleFixedSet(new int[1], 0, new FixedBitSet(1));
+            return IntArrayAtomicFieldData.EMPTY;
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
index 4198bdde0a7..6d8d2d3a487 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
@@ -32,6 +32,8 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
  */
 public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData {
 
+    public static final LongArrayAtomicFieldData EMPTY = new Empty();
+
     protected final long[] values;
     private final int numDocs;
 
@@ -47,6 +49,78 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
         return numDocs;
     }
 
+    static class Empty extends LongArrayAtomicFieldData {
+
+        Empty() {
+            super(null, 0);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return ByteValues.EMPTY;
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return ShortValues.EMPTY;
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return IntValues.EMPTY;
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return LongValues.EMPTY;
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return FloatValues.EMPTY;
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return DoubleValues.EMPTY;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return HashedBytesValues.EMPTY;
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return StringValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+    }
+
     public static class WithOrdinals extends LongArrayAtomicFieldData {
 
         private final Ordinals ordinals;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
index 28697712772..8fa135d3f3b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new LongArrayAtomicFieldData.SingleFixedSet(new long[1], 0, new FixedBitSet(1));
+            return LongArrayAtomicFieldData.EMPTY;
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
index 79bdb7da73e..ad1d1ab09f7 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -21,10 +21,12 @@ package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.GrowableWriter;
 import org.apache.lucene.util.packed.PackedInts;
 import org.elasticsearch.common.RamUsage;
 import org.elasticsearch.common.lucene.HashedBytesRef;
 import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.Ordinals;
 import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
 import org.elasticsearch.index.fielddata.util.IntArrayRef;
@@ -34,10 +36,14 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
  */
 public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
 
+    public static PagedBytesAtomicFieldData empty(int numDocs) {
+        return new Empty(numDocs);
+    }
+
     // 0 ordinal in values means no value (its null)
     private final PagedBytes.Reader bytes;
     private final PackedInts.Reader termOrdToBytesOffset;
-    private final Ordinals ordinals;
+    protected final Ordinals ordinals;
 
     private int[] hashes;
     private long size = -1;
@@ -595,4 +601,53 @@ public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptD
             }
         }
     }
+
+    static class Empty extends PagedBytesAtomicFieldData {
+
+        Empty(int numDocs) {
+            super(emptyBytes(), new GrowableWriter(1, 2, PackedInts.FASTEST).getMutable(), new EmptyOrdinals(numDocs));
+        }
+
+        static PagedBytes.Reader emptyBytes() {
+            PagedBytes bytes = new PagedBytes(1);
+            bytes.copyUsingLengthPrefix(new BytesRef());
+            return bytes.freeze(true);
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return ordinals.getNumDocs();
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return true;
+        }
+
+        @Override
+        public OrdinalsBytesValues getBytesValues() {
+            return new OrdinalsBytesValues.Empty((EmptyOrdinals) ordinals);
+        }
+
+        @Override
+        public OrdinalsHashedBytesValues getHashedBytesValues() {
+            return new HashedBytesValues.Empty((EmptyOrdinals) ordinals);
+        }
+
+        @Override
+        public OrdinalsStringValues getStringValues() {
+            return new OrdinalsStringValues.Empty((EmptyOrdinals) ordinals);
+        }
+
+        @Override
+        public ScriptDocValues.Strings getScriptValues() {
+            return ScriptDocValues.EMPTY_STRINGS;
+        }
+    }
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
index def03de6abb..f142f658ed1 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesIndexFieldData.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
 import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -79,11 +78,7 @@ public class PagedBytesIndexFieldData extends AbstractIndexFieldData<PagedBytesA
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            final PagedBytes bytes = new PagedBytes(1);
-            // 0 is reserved for "unset"
-            bytes.copyUsingLengthPrefix(new BytesRef());
-            GrowableWriter termOrdToBytesOffset = new GrowableWriter(1, 2, PackedInts.FASTEST);
-            return new PagedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
+            return PagedBytesAtomicFieldData.empty(reader.maxDoc());
         }
 
         final PagedBytes bytes = new PagedBytes(15);
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
index 9733326e028..735ff1499cd 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
@@ -32,6 +32,8 @@ import org.elasticsearch.index.fielddata.util.ShortArrayRef;
  */
 public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldData {
 
+    public static final ShortArrayAtomicFieldData EMPTY = new Empty();
+
     protected final short[] values;
     private final int numDocs;
 
@@ -47,6 +49,78 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
         return numDocs;
     }
 
+    static class Empty extends ShortArrayAtomicFieldData {
+
+        Empty() {
+            super(null, 0);
+        }
+
+        @Override
+        public ByteValues getByteValues() {
+            return ByteValues.EMPTY;
+        }
+
+        @Override
+        public ShortValues getShortValues() {
+            return ShortValues.EMPTY;
+        }
+
+        @Override
+        public IntValues getIntValues() {
+            return IntValues.EMPTY;
+        }
+
+        @Override
+        public LongValues getLongValues() {
+            return LongValues.EMPTY;
+        }
+
+        @Override
+        public FloatValues getFloatValues() {
+            return FloatValues.EMPTY;
+        }
+
+        @Override
+        public DoubleValues getDoubleValues() {
+            return DoubleValues.EMPTY;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues() {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public HashedBytesValues getHashedBytesValues() {
+            return HashedBytesValues.EMPTY;
+        }
+
+        @Override
+        public StringValues getStringValues() {
+            return StringValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+    }
+
     public static class WithOrdinals extends ShortArrayAtomicFieldData {
 
         private final Ordinals ordinals;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
index 518b4c8a5f0..8eb68365f56 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java
@@ -83,7 +83,7 @@ public class ShortArrayIndexFieldData extends AbstractIndexFieldData<ShortArrayA
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new ShortArrayAtomicFieldData.SingleFixedSet(new short[1], 0, new FixedBitSet(1));
+            return ShortArrayAtomicFieldData.EMPTY;
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
index d406f53a1e1..fd1c9d5aed0 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.test.unit.index.fielddata;
 
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
 import org.apache.lucene.search.*;
 import org.elasticsearch.index.fielddata.*;
 import org.elasticsearch.index.fielddata.util.*;
@@ -1247,4 +1250,243 @@ public abstract class NumericFieldDataTests extends StringFieldDataTests {
         floatValues.forEachValueInDoc(1, new FloatValuesVerifierProc(1).addMissing());
         floatValues.forEachValueInDoc(2, new FloatValuesVerifierProc(2).addExpected(3f));
     }
+
+    @Test
+    public void testMissingValueForAll() throws Exception {
+        fillAllMissing();
+        IndexNumericFieldData indexFieldData = getForField("value");
+        AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(0));
+
+        // long values
+
+        LongValues longValues = fieldData.getLongValues();
+
+        assertThat(longValues.isMultiValued(), equalTo(false));
+
+        assertThat(longValues.hasValue(0), equalTo(false));
+        assertThat(longValues.hasValue(1), equalTo(false));
+        assertThat(longValues.hasValue(2), equalTo(false));
+
+        assertThat(longValues.getValueMissing(0, -1), equalTo(-1l));
+        assertThat(longValues.getValueMissing(1, -1), equalTo(-1l));
+        assertThat(longValues.getValueMissing(2, -1), equalTo(-1l));
+
+        LongArrayRef longArrayRef = longValues.getValues(0);
+        assertThat(longArrayRef.size(), equalTo(0));
+
+        longArrayRef = longValues.getValues(1);
+        assertThat(longArrayRef.size(), equalTo(0));
+
+        longArrayRef = longValues.getValues(2);
+        assertThat(longArrayRef.size(), equalTo(0));
+
+        LongValues.Iter longValuesIter = longValues.getIter(0);
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(1);
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValuesIter = longValues.getIter(2);
+        assertThat(longValuesIter.hasNext(), equalTo(false));
+
+        longValues.forEachValueInDoc(0, new LongValuesVerifierProc(0).addMissing());
+        longValues.forEachValueInDoc(1, new LongValuesVerifierProc(1).addMissing());
+        longValues.forEachValueInDoc(2, new LongValuesVerifierProc(2).addMissing());
+
+        // double values
+
+        DoubleValues doubleValues = fieldData.getDoubleValues();
+
+        assertThat(doubleValues.isMultiValued(), equalTo(false));
+
+        assertThat(doubleValues.hasValue(0), equalTo(false));
+        assertThat(doubleValues.hasValue(1), equalTo(false));
+        assertThat(doubleValues.hasValue(2), equalTo(false));
+
+        assertThat(doubleValues.getValueMissing(0, -1), equalTo(-1d));
+        assertThat(doubleValues.getValueMissing(1, -1), equalTo(-1d));
+        assertThat(doubleValues.getValueMissing(2, -1), equalTo(-1d));
+
+        DoubleArrayRef doubleArrayRef = doubleValues.getValues(0);
+        assertThat(doubleArrayRef.size(), equalTo(0));
+
+        doubleArrayRef = doubleValues.getValues(1);
+        assertThat(doubleArrayRef.size(), equalTo(0));
+
+        doubleArrayRef = doubleValues.getValues(2);
+        assertThat(doubleArrayRef.size(), equalTo(0));
+
+        DoubleValues.Iter doubleValuesIter = doubleValues.getIter(0);
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(1);
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValuesIter = doubleValues.getIter(2);
+        assertThat(doubleValuesIter.hasNext(), equalTo(false));
+
+        doubleValues.forEachValueInDoc(0, new DoubleValuesVerifierProc(0).addMissing());
+        doubleValues.forEachValueInDoc(1, new DoubleValuesVerifierProc(1).addMissing());
+        doubleValues.forEachValueInDoc(2, new DoubleValuesVerifierProc(2).addMissing());
+
+        // byte values
+
+        ByteValues byteValues = fieldData.getByteValues();
+
+        assertThat(byteValues.isMultiValued(), equalTo(false));
+
+        assertThat(byteValues.hasValue(0), equalTo(false));
+        assertThat(byteValues.hasValue(1), equalTo(false));
+        assertThat(byteValues.hasValue(2), equalTo(false));
+
+        assertThat(byteValues.getValueMissing(0, (byte) -1), equalTo((byte) -1));
+        assertThat(byteValues.getValueMissing(1, (byte) -1), equalTo((byte) -1));
+        assertThat(byteValues.getValueMissing(2, (byte) -1), equalTo((byte) -1));
+
+        ByteArrayRef byteArrayRef = byteValues.getValues(0);
+        assertThat(byteArrayRef.size(), equalTo(0));
+
+        byteArrayRef = byteValues.getValues(1);
+        assertThat(byteArrayRef.size(), equalTo(0));
+
+        byteArrayRef = byteValues.getValues(2);
+        assertThat(byteArrayRef.size(), equalTo(0));
+
+        ByteValues.Iter byteValuesIter = byteValues.getIter(0);
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(1);
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValuesIter = byteValues.getIter(2);
+        assertThat(byteValuesIter.hasNext(), equalTo(false));
+
+        byteValues.forEachValueInDoc(0, new ByteValuesVerifierProc(0).addMissing());
+        byteValues.forEachValueInDoc(1, new ByteValuesVerifierProc(1).addMissing());
+        byteValues.forEachValueInDoc(2, new ByteValuesVerifierProc(2).addMissing());
+
+        // short values
+
+        ShortValues shortValues = fieldData.getShortValues();
+
+        assertThat(shortValues.isMultiValued(), equalTo(false));
+
+        assertThat(shortValues.hasValue(0), equalTo(false));
+        assertThat(shortValues.hasValue(1), equalTo(false));
+        assertThat(shortValues.hasValue(2), equalTo(false));
+
+        assertThat(shortValues.getValueMissing(0, (short) -1), equalTo((short) -1));
+        assertThat(shortValues.getValueMissing(1, (short) -1), equalTo((short) -1));
+        assertThat(shortValues.getValueMissing(2, (short) -1), equalTo((short) -1));
+
+        ShortArrayRef shortArrayRef = shortValues.getValues(0);
+        assertThat(shortArrayRef.size(), equalTo(0));
+
+        shortArrayRef = shortValues.getValues(1);
+        assertThat(shortArrayRef.size(), equalTo(0));
+
+        shortArrayRef = shortValues.getValues(2);
+        assertThat(shortArrayRef.size(), equalTo(0));
+
+        ShortValues.Iter shortValuesIter = shortValues.getIter(0);
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(1);
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValuesIter = shortValues.getIter(2);
+        assertThat(shortValuesIter.hasNext(), equalTo(false));
+
+        shortValues.forEachValueInDoc(0, new ShortValuesVerifierProc(0).addMissing());
+        shortValues.forEachValueInDoc(1, new ShortValuesVerifierProc(1).addMissing());
+        shortValues.forEachValueInDoc(2, new ShortValuesVerifierProc(2).addMissing());
+
+        // int values
+
+        IntValues intValues = fieldData.getIntValues();
+
+        assertThat(intValues.isMultiValued(), equalTo(false));
+
+        assertThat(intValues.hasValue(0), equalTo(false));
+        assertThat(intValues.hasValue(1), equalTo(false));
+        assertThat(intValues.hasValue(2), equalTo(false));
+
+        assertThat(intValues.getValueMissing(0, -1), equalTo(-1));
+        assertThat(intValues.getValueMissing(1, -1), equalTo(-1));
+        assertThat(intValues.getValueMissing(2, -1), equalTo(-1));
+
+        IntArrayRef intArrayRef = intValues.getValues(0);
+        assertThat(intArrayRef.size(), equalTo(0));
+
+        intArrayRef = intValues.getValues(1);
+        assertThat(intArrayRef.size(), equalTo(0));
+
+        intArrayRef = intValues.getValues(2);
+        assertThat(intArrayRef.size(), equalTo(0));
+
+        IntValues.Iter intValuesIter = intValues.getIter(0);
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(1);
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValuesIter = intValues.getIter(2);
+        assertThat(intValuesIter.hasNext(), equalTo(false));
+
+        intValues.forEachValueInDoc(0, new IntValuesVerifierProc(0).addMissing());
+        intValues.forEachValueInDoc(1, new IntValuesVerifierProc(1).addMissing());
+        intValues.forEachValueInDoc(2, new IntValuesVerifierProc(2).addMissing());
+
+        // float
+
+        FloatValues floatValues = fieldData.getFloatValues();
+
+        assertThat(floatValues.isMultiValued(), equalTo(false));
+
+        assertThat(floatValues.hasValue(0), equalTo(false));
+        assertThat(floatValues.hasValue(1), equalTo(false));
+        assertThat(floatValues.hasValue(2), equalTo(false));
+
+        assertThat(floatValues.getValueMissing(0, -1), equalTo(-1f));
+        assertThat(floatValues.getValueMissing(1, -1), equalTo(-1f));
+        assertThat(floatValues.getValueMissing(2, -1), equalTo(-1f));
+
+        FloatArrayRef floatArrayRef = floatValues.getValues(0);
+        assertThat(floatArrayRef.size(), equalTo(0));
+
+        floatArrayRef = floatValues.getValues(1);
+        assertThat(floatArrayRef.size(), equalTo(0));
+
+        floatArrayRef = floatValues.getValues(2);
+        assertThat(floatArrayRef.size(), equalTo(0));
+
+        FloatValues.Iter floatValuesIter = floatValues.getIter(0);
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(1);
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValuesIter = floatValues.getIter(2);
+        assertThat(floatValuesIter.hasNext(), equalTo(false));
+
+        floatValues.forEachValueInDoc(0, new FloatValuesVerifierProc(0).addMissing());
+        floatValues.forEachValueInDoc(1, new FloatValuesVerifierProc(1).addMissing());
+        floatValues.forEachValueInDoc(2, new FloatValuesVerifierProc(2).addMissing());
+    }
+
+    protected void fillAllMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
 }
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
index 799458874a3..dbc002878f4 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
@@ -624,4 +624,125 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
         stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
     }
+
+    public void testMissingValueForAll() throws Exception {
+        fillAllMissing();
+        IndexFieldData indexFieldData = getForField("value");
+        AtomicFieldData fieldData = indexFieldData.load(refreshReader());
+
+        assertThat(fieldData.getNumDocs(), equalTo(3));
+
+        BytesValues bytesValues = fieldData.getBytesValues();
+
+        assertThat(bytesValues.isMultiValued(), equalTo(false));
+
+        assertThat(bytesValues.hasValue(0), equalTo(false));
+        assertThat(bytesValues.hasValue(1), equalTo(false));
+        assertThat(bytesValues.hasValue(2), equalTo(false));
+
+        assertThat(bytesValues.getValue(0), nullValue());
+        assertThat(bytesValues.getValue(1), nullValue());
+        assertThat(bytesValues.getValue(2), nullValue());
+
+        BytesRef bytesRef = new BytesRef();
+        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef()));
+        assertThat(bytesRef, equalTo(new BytesRef()));
+        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
+        assertThat(bytesRef, equalTo(new BytesRef()));
+        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef()));
+        assertThat(bytesRef, equalTo(new BytesRef()));
+
+
+        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
+        assertThat(bytesRefArrayRef.size(), equalTo(0));
+
+        bytesRefArrayRef = bytesValues.getValues(1);
+        assertThat(bytesRefArrayRef.size(), equalTo(0));
+
+        bytesRefArrayRef = bytesValues.getValues(2);
+        assertThat(bytesRefArrayRef.size(), equalTo(0));
+
+        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIter(1);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValuesIter = bytesValues.getIter(2);
+        assertThat(bytesValuesIter.hasNext(), equalTo(false));
+
+        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addMissing());
+        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
+        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addMissing());
+
+        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
+
+        assertThat(hashedBytesValues.hasValue(0), equalTo(false));
+        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
+        assertThat(hashedBytesValues.hasValue(2), equalTo(false));
+
+        assertThat(hashedBytesValues.getValue(0), nullValue());
+        assertThat(hashedBytesValues.getValue(1), nullValue());
+        assertThat(hashedBytesValues.getValue(2), nullValue());
+
+        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIter(1);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValuesIter = hashedBytesValues.getIter(2);
+        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
+
+        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addMissing());
+        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
+        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addMissing());
+
+        StringValues stringValues = fieldData.getStringValues();
+
+        assertThat(stringValues.hasValue(0), equalTo(false));
+        assertThat(stringValues.hasValue(1), equalTo(false));
+        assertThat(stringValues.hasValue(2), equalTo(false));
+
+        assertThat(stringValues.getValue(0), nullValue());
+        assertThat(stringValues.getValue(1), nullValue());
+        assertThat(stringValues.getValue(2), nullValue());
+
+        StringArrayRef stringArrayRef;
+        stringArrayRef = stringValues.getValues(0);
+        assertThat(stringArrayRef.size(), equalTo(0));
+
+        stringArrayRef = stringValues.getValues(1);
+        assertThat(stringArrayRef.size(), equalTo(0));
+
+        stringArrayRef = stringValues.getValues(2);
+        assertThat(stringArrayRef.size(), equalTo(0));
+
+        StringValues.Iter stringValuesIter = stringValues.getIter(0);
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(1);
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValuesIter = stringValues.getIter(2);
+        assertThat(stringValuesIter.hasNext(), equalTo(false));
+
+        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addMissing());
+        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
+        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addMissing());
+    }
+
+    protected void fillAllMissing() throws Exception {
+        Document d = new Document();
+        d.add(new StringField("_id", "1", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "2", Field.Store.NO));
+        writer.addDocument(d);
+
+        d = new Document();
+        d.add(new StringField("_id", "3", Field.Store.NO));
+        writer.addDocument(d);
+    }
 }

From 27bfb341ff0252a732892c57b2fddf9badb0959e Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 15:49:52 +0100
Subject: [PATCH 34/35] better logging on missing format, and allow to
 configure format on a type on the index level

---
 .../index/fielddata/IndexFieldDataService.java         | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index ce8b5fbd61b..f8fcd863c9b 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -121,6 +121,16 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                     IndexFieldData.Builder builder = null;
                     if (type.getFormat() != null) {
                         builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), type.getFormat()));
+                        if (builder == null) {
+                            logger.warn("failed to find format [" + type.getFormat() + "] for field [" + fieldNames.fullName() + "], will use default");
+                        }
+                    }
+                    String format = indexSettings.get("index.fielddata.type." + type.getType() + ".format", null);
+                    if (format != null) {
+                        builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), type.getFormat()));
+                        if (builder == null) {
+                            logger.warn("failed to find index level type format [" + format + "] for field [" + fieldNames.fullName() + "], will use default");
+                        }
                     }
                     if (builder == null) {
                         builder = buildersByType.get(type.getType());

From c295211a85722dd8ad838c94a00c18a42347ff0c Mon Sep 17 00:00:00 2001
From: Shay Banon <kimchy@gmail.com>
Date: Tue, 22 Jan 2013 16:13:24 +0100
Subject: [PATCH 35/35] final move to new field data

---
 .../index/cache/IndexCacheModule.java         |   2 -
 .../cache/field/data/FieldDataCache.java      |  51 ---
 .../field/data/FieldDataCacheModule.java      |  48 ---
 .../field/data/none/NoneFieldDataCache.java   |  88 -----
 .../data/resident/ResidentFieldDataCache.java | 130 --------
 .../field/data/soft/SoftFieldDataCache.java   |  68 ----
 .../AbstractConcurrentMapFieldDataCache.java  | 150 ---------
 .../index/field/data/DocFieldData.java        |  68 ----
 .../index/field/data/FieldData.java           | 114 -------
 .../index/field/data/FieldDataType.java       |  63 ----
 .../index/field/data/NumericDocFieldData.java |  58 ----
 .../index/field/data/NumericFieldData.java    | 123 -------
 .../field/data/bytes/ByteDocFieldData.java    |  40 ---
 .../index/field/data/bytes/ByteFieldData.java | 165 ----------
 .../data/bytes/ByteFieldDataComparator.java   |  77 -----
 .../bytes/ByteFieldDataMissingComparator.java |  90 -----
 .../field/data/bytes/ByteFieldDataType.java   |  93 ------
 .../data/bytes/MultiValueByteFieldData.java   | 239 --------------
 .../data/bytes/SingleValueByteFieldData.java  | 159 ---------
 .../data/doubles/DoubleDocFieldData.java      |  40 ---
 .../field/data/doubles/DoubleFieldData.java   | 165 ----------
 .../doubles/DoubleFieldDataComparator.java    |  92 ------
 .../DoubleFieldDataMissingComparator.java     | 104 ------
 .../data/doubles/DoubleFieldDataType.java     |  93 ------
 .../doubles/MultiValueDoubleFieldData.java    | 208 ------------
 .../doubles/SingleValueDoubleFieldData.java   | 145 --------
 .../field/data/floats/FloatDocFieldData.java  |  40 ---
 .../field/data/floats/FloatFieldData.java     | 165 ----------
 .../data/floats/FloatFieldDataComparator.java |  96 ------
 .../FloatFieldDataMissingComparator.java      | 108 ------
 .../field/data/floats/FloatFieldDataType.java |  93 ------
 .../data/floats/MultiValueFloatFieldData.java | 239 --------------
 .../floats/SingleValueFloatFieldData.java     | 159 ---------
 .../field/data/ints/IntDocFieldData.java      |  40 ---
 .../index/field/data/ints/IntFieldData.java   | 165 ----------
 .../data/ints/IntFieldDataComparator.java     | 102 ------
 .../ints/IntFieldDataMissingComparator.java   | 114 -------
 .../field/data/ints/IntFieldDataType.java     |  93 ------
 .../data/ints/MultiValueIntFieldData.java     | 239 --------------
 .../data/ints/SingleValueIntFieldData.java    | 159 ---------
 .../field/data/longs/LongDocFieldData.java    |  49 ---
 .../index/field/data/longs/LongFieldData.java | 195 -----------
 .../data/longs/LongFieldDataComparator.java   |  98 ------
 .../longs/LongFieldDataMissingComparator.java | 110 -------
 .../field/data/longs/LongFieldDataType.java   |  93 ------
 .../data/longs/MultiValueLongFieldData.java   | 311 ------------------
 .../data/longs/SingleValueLongFieldData.java  | 202 ------------
 .../data/shorts/MultiValueShortFieldData.java | 248 --------------
 .../field/data/shorts/ShortDocFieldData.java  |  40 ---
 .../field/data/shorts/ShortFieldData.java     | 164 ---------
 .../data/shorts/ShortFieldDataComparator.java |  78 -----
 .../ShortFieldDataMissingComparator.java      |  91 -----
 .../field/data/shorts/ShortFieldDataType.java |  93 ------
 .../shorts/SingleValueShortFieldData.java     | 159 ---------
 .../strings/MultiValueStringFieldData.java    | 141 --------
 .../strings/SingleValueStringFieldData.java   |  98 ------
 .../data/strings/StringDocFieldData.java      |  61 ----
 .../field/data/strings/StringFieldData.java   | 120 -------
 .../data/strings/StringFieldDataType.java     |  58 ----
 .../StringOrdValFieldDataComparator.java      | 224 -------------
 .../strings/StringValFieldDataComparator.java | 118 -------
 .../field/data/support/FieldDataLoader.java   | 110 -------
 .../support/NumericFieldDataComparator.java   |  54 ---
 .../fielddata/IndexFieldDataService.java      |   2 +-
 .../DoubleScriptDataComparator.java}          |  14 +-
 .../StringScriptDataComparator.java}          |  14 +-
 .../index/mapper/FieldMapper.java             |   4 +-
 .../mapper/core/AbstractFieldMapper.java      |   6 -
 .../index/mapper/core/BinaryFieldMapper.java  |   2 +-
 .../index/mapper/core/BooleanFieldMapper.java |   7 +-
 .../index/mapper/core/ByteFieldMapper.java    |  11 +-
 .../index/mapper/core/DateFieldMapper.java    |  12 +-
 .../index/mapper/core/DoubleFieldMapper.java  |  11 +-
 .../index/mapper/core/FloatFieldMapper.java   |  11 +-
 .../index/mapper/core/IntegerFieldMapper.java |   8 +-
 .../index/mapper/core/LongFieldMapper.java    |  11 +-
 .../index/mapper/core/NumberFieldMapper.java  |   4 -
 .../index/mapper/core/ShortFieldMapper.java   |   8 +-
 .../index/mapper/core/StringFieldMapper.java  |   2 +-
 .../mapper/geo/GeoPointDocFieldData.java      | 135 --------
 .../index/mapper/geo/GeoPointFieldData.java   | 205 ------------
 .../mapper/geo/GeoPointFieldDataType.java     |  57 ----
 .../index/mapper/geo/GeoPointFieldMapper.java |   8 +-
 .../index/mapper/geo/GeoShapeFieldMapper.java |   3 +-
 .../geo/MultiValueGeoPointFieldData.java      | 254 --------------
 .../geo/SingleValueGeoPointFieldData.java     | 159 ---------
 .../index/mapper/internal/AllFieldMapper.java |   2 +-
 .../mapper/internal/BoostFieldMapper.java     |   9 +-
 .../index/mapper/internal/IdFieldMapper.java  |   2 +-
 .../mapper/internal/IndexFieldMapper.java     |   2 +-
 .../mapper/internal/ParentFieldMapper.java    |   2 +-
 .../mapper/internal/RoutingFieldMapper.java   |   2 +-
 .../mapper/internal/SourceFieldMapper.java    |   3 +-
 .../mapper/internal/TypeFieldMapper.java      |   2 +-
 .../index/mapper/internal/UidFieldMapper.java |   2 +-
 .../index/mapper/ip/IpFieldMapper.java        |   8 +-
 .../search/geo/GeoDistanceDataComparator.java | 201 -----------
 .../search/lookup/DocLookup.java              |  10 -
 .../search/sort/ScriptSortParser.java         |   8 +-
 .../data/doubles/DoubleFieldDataTests.java    | 165 ----------
 .../data/floats/FloatFieldDataTests.java      | 165 ----------
 .../field/data/ints/IntFieldDataTests.java    | 157 ---------
 .../field/data/longs/LongFieldDataTests.java  | 165 ----------
 .../data/shorts/ShortFieldDataTests.java      | 163 ---------
 .../data/strings/StringFieldDataTests.java    | 160 ---------
 .../PathMatchDynamicTemplateTests.java        |   3 -
 106 files changed, 58 insertions(+), 9521 deletions(-)
 delete mode 100644 src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java
 delete mode 100644 src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCacheModule.java
 delete mode 100644 src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java
 delete mode 100644 src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java
 delete mode 100644 src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java
 delete mode 100644 src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/DocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/FieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/FieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/NumericDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/bytes/ByteDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/doubles/DoubleDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/doubles/SingleValueDoubleFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/floats/FloatDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/ints/IntDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/longs/LongDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/shorts/ShortDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/strings/StringDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/strings/StringFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/strings/StringOrdValFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/strings/StringValFieldDataComparator.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/support/FieldDataLoader.java
 delete mode 100644 src/main/java/org/elasticsearch/index/field/data/support/NumericFieldDataComparator.java
 rename src/main/java/org/elasticsearch/index/{field/function/sort/DoubleFieldsFunctionDataComparator.java => fielddata/fieldcomparator/DoubleScriptDataComparator.java} (85%)
 rename src/main/java/org/elasticsearch/index/{field/function/sort/StringFieldsFunctionDataComparator.java => fielddata/fieldcomparator/StringScriptDataComparator.java} (85%)
 delete mode 100644 src/main/java/org/elasticsearch/index/mapper/geo/GeoPointDocFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldDataType.java
 delete mode 100644 src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/mapper/geo/SingleValueGeoPointFieldData.java
 delete mode 100644 src/main/java/org/elasticsearch/index/search/geo/GeoDistanceDataComparator.java
 delete mode 100644 src/test/java/org/elasticsearch/test/unit/index/field/data/doubles/DoubleFieldDataTests.java
 delete mode 100644 src/test/java/org/elasticsearch/test/unit/index/field/data/floats/FloatFieldDataTests.java
 delete mode 100644 src/test/java/org/elasticsearch/test/unit/index/field/data/ints/IntFieldDataTests.java
 delete mode 100644 src/test/java/org/elasticsearch/test/unit/index/field/data/longs/LongFieldDataTests.java
 delete mode 100644 src/test/java/org/elasticsearch/test/unit/index/field/data/shorts/ShortFieldDataTests.java
 delete mode 100644 src/test/java/org/elasticsearch/test/unit/index/field/data/strings/StringFieldDataTests.java

diff --git a/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java b/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java
index c95535c2b70..4b090477069 100644
--- a/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java
+++ b/src/main/java/org/elasticsearch/index/cache/IndexCacheModule.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.cache;
 
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.cache.field.data.FieldDataCacheModule;
 import org.elasticsearch.index.cache.filter.FilterCacheModule;
 import org.elasticsearch.index.cache.id.IdCacheModule;
 import org.elasticsearch.index.cache.query.parser.QueryParserCacheModule;
@@ -40,7 +39,6 @@ public class IndexCacheModule extends AbstractModule {
     @Override
     protected void configure() {
         new FilterCacheModule(settings).configure(binder());
-        new FieldDataCacheModule(settings).configure(binder());
         new IdCacheModule(settings).configure(binder());
         new QueryParserCacheModule(settings).configure(binder());
 
diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java
deleted file mode 100644
index a1ba5b4cc41..00000000000
--- a/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.cache.field.data;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.elasticsearch.common.component.CloseableComponent;
-import org.elasticsearch.index.IndexComponent;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public interface FieldDataCache extends IndexComponent, CloseableComponent {
-
-    FieldData cache(FieldDataType type, AtomicReader reader, String fieldName) throws IOException;
-
-    String type();
-
-    void clear(String reason, String fieldName);
-
-    void clear(String reason);
-
-    void clear(IndexReader reader);
-
-    long evictions();
-
-    long sizeInBytes();
-
-    long sizeInBytes(String fieldName);
-}
diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCacheModule.java b/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCacheModule.java
deleted file mode 100644
index 6abca05d508..00000000000
--- a/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCacheModule.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.cache.field.data;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Scopes;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.cache.field.data.resident.ResidentFieldDataCache;
-
-/**
- *
- */
-public class FieldDataCacheModule extends AbstractModule {
-
-    public static final class FieldDataCacheSettings {
-        public static final String FIELD_DATA_CACHE_TYPE = "index.cache.field.type";
-    }
-
-    private final Settings settings;
-
-    public FieldDataCacheModule(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    protected void configure() {
-        bind(FieldDataCache.class)
-                .to(settings.getAsClass(FieldDataCacheSettings.FIELD_DATA_CACHE_TYPE, ResidentFieldDataCache.class, "org.elasticsearch.index.cache.field.data.", "FieldDataCache"))
-                .in(Scopes.SINGLETON);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java
deleted file mode 100644
index 04c404f0343..00000000000
--- a/src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.cache.field.data.none;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.elasticsearch.ElasticSearchException;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.AbstractIndexComponent;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.settings.IndexSettings;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class NoneFieldDataCache extends AbstractIndexComponent implements FieldDataCache {
-
-    @Inject
-    public NoneFieldDataCache(Index index, @IndexSettings Settings indexSettings) {
-        super(index, indexSettings);
-        logger.debug("Using no field cache");
-    }
-
-    @Override
-    public FieldData cache(FieldDataType type, AtomicReader reader, String fieldName) throws IOException {
-        return FieldData.load(type, reader, fieldName);
-    }
-
-    @Override
-    public String type() {
-        return "none";
-    }
-
-    @Override
-    public void clear(String reason, String fieldName) {
-
-    }
-
-    @Override
-    public void clear(String reason) {
-    }
-
-    @Override
-    public void clear(IndexReader reader) {
-    }
-
-    @Override
-    public void close() throws ElasticSearchException {
-    }
-
-    @Override
-    public long sizeInBytes() {
-        return 0;
-    }
-
-    @Override
-    public long sizeInBytes(String fieldName) {
-        return 0;
-    }
-
-    @Override
-    public long evictions() {
-        return 0;
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java
deleted file mode 100644
index 6bd2effb578..00000000000
--- a/src/main/java/org/elasticsearch/index/cache/field/data/resident/ResidentFieldDataCache.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.cache.field.data.resident;
-
-import com.google.common.base.Objects;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import org.elasticsearch.ElasticSearchException;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.field.data.support.AbstractConcurrentMapFieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.settings.IndexSettings;
-import org.elasticsearch.index.settings.IndexSettingsService;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public class ResidentFieldDataCache extends AbstractConcurrentMapFieldDataCache implements RemovalListener<String, FieldData> {
-
-    private final IndexSettingsService indexSettingsService;
-
-    private volatile int maxSize;
-    private volatile TimeValue expire;
-
-    private final CounterMetric evictions = new CounterMetric();
-
-    private final ApplySettings applySettings = new ApplySettings();
-
-    @Inject
-    public ResidentFieldDataCache(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService) {
-        super(index, indexSettings);
-        this.indexSettingsService = indexSettingsService;
-
-        this.maxSize = indexSettings.getAsInt("index.cache.field.max_size", componentSettings.getAsInt("max_size", -1));
-        this.expire = indexSettings.getAsTime("index.cache.field.expire", componentSettings.getAsTime("expire", null));
-        logger.debug("using [resident] field cache with max_size [{}], expire [{}]", maxSize, expire);
-
-        indexSettingsService.addListener(applySettings);
-    }
-
-    @Override
-    public void close() throws ElasticSearchException {
-        indexSettingsService.removeListener(applySettings);
-        super.close();
-    }
-
-    @Override
-    protected Cache<String, FieldData> buildFieldDataMap() {
-        CacheBuilder<String, FieldData> cacheBuilder = CacheBuilder.newBuilder().removalListener(this);
-        if (maxSize != -1) {
-            cacheBuilder.maximumSize(maxSize);
-        }
-        if (expire != null) {
-            cacheBuilder.expireAfterAccess(expire.nanos(), TimeUnit.NANOSECONDS);
-        }
-        return cacheBuilder.build();
-    }
-
-    @Override
-    public String type() {
-        return "resident";
-    }
-
-    @Override
-    public long evictions() {
-        return evictions.count();
-    }
-
-    @Override
-    public void onRemoval(RemovalNotification<String, FieldData> removalNotification) {
-        if (removalNotification.wasEvicted()) {
-            evictions.inc();
-        }
-    }
-
-    static {
-        IndexMetaData.addDynamicSettings(
-                "index.cache.field.max_size",
-                "index.cache.field.expire"
-        );
-    }
-
-    class ApplySettings implements IndexSettingsService.Listener {
-        @Override
-        public void onRefreshSettings(Settings settings) {
-            int maxSize = settings.getAsInt("index.cache.field.max_size", ResidentFieldDataCache.this.maxSize);
-            TimeValue expire = settings.getAsTime("index.cache.field.expire", ResidentFieldDataCache.this.expire);
-            boolean changed = false;
-            if (maxSize != ResidentFieldDataCache.this.maxSize) {
-                logger.info("updating index.cache.field.max_size from [{}] to [{}]", ResidentFieldDataCache.this.maxSize, maxSize);
-                changed = true;
-                ResidentFieldDataCache.this.maxSize = maxSize;
-            }
-            if (!Objects.equal(expire, ResidentFieldDataCache.this.expire)) {
-                logger.info("updating index.cache.field.expire from [{}] to [{}]", ResidentFieldDataCache.this.expire, expire);
-                changed = true;
-                ResidentFieldDataCache.this.expire = expire;
-            }
-            if (changed) {
-                clear("update_settings");
-            }
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java
deleted file mode 100644
index 38f83fd6105..00000000000
--- a/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.cache.field.data.soft;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.field.data.support.AbstractConcurrentMapFieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.settings.IndexSettings;
-
-/**
- *
- */
-public class SoftFieldDataCache extends AbstractConcurrentMapFieldDataCache implements RemovalListener<String, FieldData> {
-
-    private final CounterMetric evictions = new CounterMetric();
-
-    @Inject
-    public SoftFieldDataCache(Index index, @IndexSettings Settings indexSettings) {
-        super(index, indexSettings);
-    }
-
-    @Override
-    protected Cache<String, FieldData> buildFieldDataMap() {
-        CacheBuilder<String, FieldData> cacheBuilder = CacheBuilder.newBuilder().softValues().removalListener(this);
-        return cacheBuilder.build();
-    }
-
-    @Override
-    public long evictions() {
-        return evictions.count();
-    }
-
-    @Override
-    public String type() {
-        return "soft";
-    }
-
-    @Override
-    public void onRemoval(RemovalNotification<String, FieldData> removalNotification) {
-        if (removalNotification.wasEvicted()) {
-            evictions.inc();
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java
deleted file mode 100644
index 7edb9f90ac6..00000000000
--- a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.cache.field.data.support;
-
-import com.google.common.cache.Cache;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.SegmentReader;
-import org.elasticsearch.ElasticSearchException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.index.AbstractIndexComponent;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.settings.IndexSettings;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexComponent implements FieldDataCache, SegmentReader.CoreClosedListener {
-
-    private final ConcurrentMap<Object, Cache<String, FieldData>> cache;
-
-    private final Object creationMutex = new Object();
-
-    protected AbstractConcurrentMapFieldDataCache(Index index, @IndexSettings Settings indexSettings) {
-        super(index, indexSettings);
-        this.cache = ConcurrentCollections.newConcurrentMap();
-    }
-
-    @Override
-    public void close() throws ElasticSearchException {
-        clear("close");
-    }
-
-    @Override
-    public void clear(String reason, String fieldName) {
-        logger.debug("clearing field [{}] cache, reason [{}]", fieldName, reason);
-        for (Map.Entry<Object, Cache<String, FieldData>> entry : cache.entrySet()) {
-            entry.getValue().invalidate(fieldName);
-        }
-    }
-
-    @Override
-    public void clear(String reason) {
-        logger.debug("full cache clear, reason [{}]", reason);
-        cache.clear();
-    }
-
-    @Override
-    public void onClose(SegmentReader owner) {
-        clear(owner);
-    }
-
-    @Override
-    public void clear(IndexReader reader) {
-        cache.remove(reader.getCoreCacheKey());
-    }
-
-    @Override
-    public long sizeInBytes() {
-        // the overhead of the map is not really relevant...
-        long sizeInBytes = 0;
-        for (Cache<String, FieldData> map : cache.values()) {
-            for (FieldData fieldData : map.asMap().values()) {
-                sizeInBytes += fieldData.sizeInBytes();
-            }
-        }
-        return sizeInBytes;
-    }
-
-    @Override
-    public long sizeInBytes(String fieldName) {
-        long sizeInBytes = 0;
-        for (Cache<String, FieldData> map : cache.values()) {
-            FieldData fieldData = map.getIfPresent(fieldName);
-            if (fieldData != null) {
-                sizeInBytes += fieldData.sizeInBytes();
-            }
-        }
-        return sizeInBytes;
-    }
-
-    @Override
-    public FieldData cache(FieldDataType type, AtomicReader reader, String fieldName) throws IOException {
-        Cache<String, FieldData> fieldDataCache = cache.get(reader.getCoreCacheKey());
-        if (fieldDataCache == null) {
-            synchronized (creationMutex) {
-                fieldDataCache = cache.get(reader.getCoreCacheKey());
-                if (fieldDataCache == null) {
-                    fieldDataCache = buildFieldDataMap();
-                    if (reader instanceof SegmentReader) {
-                        ((SegmentReader) reader).addCoreClosedListener(this);
-                    }
-                    cache.put(reader.getCoreCacheKey(), fieldDataCache);
-                }
-            }
-        }
-        FieldData fieldData = fieldDataCache.getIfPresent(fieldName);
-        if (fieldData == null) {
-            synchronized (fieldDataCache) {
-                fieldData = fieldDataCache.getIfPresent(fieldName);
-                if (fieldData == null) {
-                    try {
-                        long time = System.nanoTime();
-                        fieldData = FieldData.load(type, reader, fieldName);
-                        fieldDataCache.put(fieldName, fieldData);
-                        long took = System.nanoTime() - time;
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("loaded field [{}] for reader [{}], took [{}], took_millis [{}]", fieldName, reader, TimeValue.timeValueNanos(took), TimeUnit.NANOSECONDS.toMillis(took));
-                        }
-                    } catch (OutOfMemoryError e) {
-                        logger.warn("loading field [" + fieldName + "] caused out of memory failure", e);
-                        final OutOfMemoryError outOfMemoryError = new OutOfMemoryError("loading field [" + fieldName + "] caused out of memory failure");
-                        outOfMemoryError.initCause(e);
-                        throw outOfMemoryError;
-                    }
-                }
-            }
-        }
-        return fieldData;
-    }
-
-    protected abstract Cache<String, FieldData> buildFieldDataMap();
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/DocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/DocFieldData.java
deleted file mode 100644
index c6cc070a8f4..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/DocFieldData.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data;
-
-import org.apache.lucene.util.BytesRef;
-
-/**
- *
- */
-public abstract class DocFieldData<T extends FieldData> {
-
-    protected final T fieldData;
-
-    protected int docId;
-
-    protected DocFieldData(T fieldData) {
-        this.fieldData = fieldData;
-    }
-
-    void setDocId(int docId) {
-        this.docId = docId;
-    }
-
-    public String getFieldName() {
-        return fieldData.fieldName();
-    }
-
-    public boolean isEmpty() {
-        return !fieldData.hasValue(docId);
-    }
-
-    public String stringValue() {
-        BytesRef val = fieldData.stringValue(docId);
-        if (val == null) {
-            return null;
-        }
-        return val.utf8ToString();
-    }
-
-    public String getStringValue() {
-        return stringValue();
-    }
-
-    public FieldDataType getType() {
-        return fieldData.type();
-    }
-
-    public boolean isMultiValued() {
-        return fieldData.multiValued();
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/FieldData.java b/src/main/java/org/elasticsearch/index/field/data/FieldData.java
deleted file mode 100644
index acb7b96e9da..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/FieldData.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-
-import java.io.IOException;
-
-/**
- *
- */
-// General TODOs on FieldData
-// TODO Optimize the order (both int[] and int[][] when they are sparse, create an Order abstraction)
-public abstract class FieldData<Doc extends DocFieldData> {
-
-    private final ThreadLocal<ThreadLocals.CleanableValue<Doc>> cachedDocFieldData = new ThreadLocal<ThreadLocals.CleanableValue<Doc>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<Doc> initialValue() {
-            return new ThreadLocals.CleanableValue<Doc>(createFieldData());
-        }
-    };
-
-    private final String fieldName;
-
-    private long sizeInBytes = -1;
-
-    protected FieldData(String fieldName) {
-        this.fieldName = fieldName;
-    }
-
-    /**
-     * The field name of this field data.
-     */
-    public final String fieldName() {
-        return fieldName;
-    }
-
-    public Doc docFieldData(int docId) {
-        Doc docFieldData = cachedDocFieldData.get().get();
-        docFieldData.setDocId(docId);
-        return docFieldData;
-    }
-
-    public long sizeInBytes() {
-        if (sizeInBytes == -1) {
-            sizeInBytes = computeSizeInBytes();
-        }
-        return sizeInBytes;
-    }
-
-    protected abstract long computeSizeInBytes();
-
-    protected abstract Doc createFieldData();
-
-    /**
-     * Is the field data a multi valued one (has multiple values / terms per document id) or not.
-     */
-    public abstract boolean multiValued();
-
-    /**
-     * Is there a value associated with this document id.
-     */
-    public abstract boolean hasValue(int docId);
-
-    public abstract BytesRef stringValue(int docId);
-
-    public abstract void forEachValue(StringValueProc proc);
-
-    public static interface StringValueProc {
-        void onValue(BytesRef value);
-    }
-
-    public abstract void forEachValueInDoc(int docId, StringValueInDocProc proc);
-
-    public static interface StringValueInDocProc {
-        void onValue(int docId, BytesRef value);
-
-        void onMissing(int docId);
-    }
-
-    public abstract void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc);
-
-    public static interface OrdinalInDocProc {
-        void onOrdinal(int docId, int ordinal);
-    }
-
-    /**
-     * The type of this field data.
-     */
-    public abstract FieldDataType type();
-
-    public static FieldData load(FieldDataType type, AtomicReader reader, String fieldName) throws IOException {
-        return type.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/FieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/FieldDataType.java
deleted file mode 100644
index d2d724f2a4b..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/FieldDataType.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparatorSource;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.bytes.ByteFieldDataType;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldDataType;
-import org.elasticsearch.index.field.data.floats.FloatFieldDataType;
-import org.elasticsearch.index.field.data.ints.IntFieldDataType;
-import org.elasticsearch.index.field.data.longs.LongFieldDataType;
-import org.elasticsearch.index.field.data.shorts.ShortFieldDataType;
-import org.elasticsearch.index.field.data.strings.StringFieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public interface FieldDataType<T extends FieldData> {
-
-    public static final class DefaultTypes {
-        public static final StringFieldDataType STRING = new StringFieldDataType();
-        public static final ByteFieldDataType BYTE = new ByteFieldDataType();
-        public static final ShortFieldDataType SHORT = new ShortFieldDataType();
-        public static final IntFieldDataType INT = new IntFieldDataType();
-        public static final LongFieldDataType LONG = new LongFieldDataType();
-        public static final FloatFieldDataType FLOAT = new FloatFieldDataType();
-        public static final DoubleFieldDataType DOUBLE = new DoubleFieldDataType();
-    }
-
-    ExtendedFieldComparatorSource newFieldComparatorSource(FieldDataCache cache, @Nullable String missing);
-
-    T load(AtomicReader reader, String fieldName) throws IOException;
-
-    // we need this extended source we we have custom comparators to reuse our field data
-    // in this case, we need to reduce type that will be used when search results are reduced
-    // on another node (we don't have the custom source them...)
-    public abstract class ExtendedFieldComparatorSource extends FieldComparatorSource {
-
-        public abstract SortField.Type reducedType();
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/NumericDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/NumericDocFieldData.java
deleted file mode 100644
index c1783c01e61..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/NumericDocFieldData.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data;
-
-/**
- *
- */
-public class NumericDocFieldData<T extends NumericFieldData> extends DocFieldData<T> {
-
-    public NumericDocFieldData(T fieldData) {
-        super(fieldData);
-    }
-
-    public int getIntValue() {
-        return fieldData.intValue(docId);
-    }
-
-    public long getLongValue() {
-        return fieldData.longValue(docId);
-    }
-
-    public float getFloatValue() {
-        return fieldData.floatValue(docId);
-    }
-
-    public double getDoubleValue() {
-        return fieldData.doubleValue(docId);
-    }
-
-    public short getShortValue() {
-        return fieldData.shortValue(docId);
-    }
-
-    public byte getByteValue() {
-        return fieldData.byteValue(docId);
-    }
-
-    public double[] getDoubleValues() {
-        return fieldData.doubleValues(docId);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java b/src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java
deleted file mode 100644
index 42acb4f82d5..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/NumericFieldData.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data;
-
-/**
- *
- */
-public abstract class NumericFieldData<Doc extends NumericDocFieldData> extends FieldData<Doc> {
-
-    protected NumericFieldData(String fieldName) {
-        super(fieldName);
-    }
-
-    /**
-     * Returns the value of the specified number as an <code>int</code>.
-     * This may involve rounding or truncation.
-     *
-     * @return the numeric value represented by this object after conversion
-     *         to type <code>int</code>.
-     */
-    public abstract int intValue(int docId);
-
-    /**
-     * Returns the value of the specified number as a <code>long</code>.
-     * This may involve rounding or truncation.
-     *
-     * @return the numeric value represented by this object after conversion
-     *         to type <code>long</code>.
-     */
-    public abstract long longValue(int docId);
-
-    /**
-     * Returns the value of the specified number as a <code>float</code>.
-     * This may involve rounding.
-     *
-     * @return the numeric value represented by this object after conversion
-     *         to type <code>float</code>.
-     */
-    public abstract float floatValue(int docId);
-
-    /**
-     * Returns the value of the specified number as a <code>double</code>.
-     * This may involve rounding.
-     *
-     * @return the numeric value represented by this object after conversion
-     *         to type <code>double</code>.
-     */
-    public abstract double doubleValue(int docId);
-
-    /**
-     * Returns the value of the specified number as a <code>byte</code>.
-     * This may involve rounding or truncation.
-     *
-     * @return the numeric value represented by this object after conversion
-     *         to type <code>byte</code>.
-     */
-    public byte byteValue(int docId) {
-        return (byte) intValue(docId);
-    }
-
-    /**
-     * Returns the value of the specified number as a <code>short</code>.
-     * This may involve rounding or truncation.
-     *
-     * @return the numeric value represented by this object after conversion
-     *         to type <code>short</code>.
-     */
-    public short shortValue(int docId) {
-        return (short) intValue(docId);
-    }
-
-    @Override
-    public Doc docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    public abstract double[] doubleValues(int docId);
-
-    public abstract void forEachValueInDoc(int docId, DoubleValueInDocProc proc);
-
-    public abstract void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc);
-
-    public abstract void forEachValueInDoc(int docId, LongValueInDocProc proc);
-
-    public abstract void forEachValueInDoc(int docId, MissingLongValueInDocProc proc);
-
-    public static interface DoubleValueInDocProc {
-        void onValue(int docId, double value);
-    }
-
-    public static interface LongValueInDocProc {
-        void onValue(int docId, long value);
-    }
-
-    public static interface MissingLongValueInDocProc {
-        void onValue(int docId, long value);
-
-        void onMissing(int docId);
-    }
-
-    public static interface MissingDoubleValueInDocProc {
-        void onValue(int docId, double value);
-
-        void onMissing(int docId);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/bytes/ByteDocFieldData.java
deleted file mode 100644
index 6a6fac8519a..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteDocFieldData.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.bytes;
-
-import org.elasticsearch.index.field.data.NumericDocFieldData;
-
-/**
- *
- */
-public class ByteDocFieldData extends NumericDocFieldData<ByteFieldData> {
-
-    public ByteDocFieldData(ByteFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public byte getValue() {
-        return fieldData.value(docId);
-    }
-
-    public byte[] getValues() {
-        return fieldData.values(docId);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java b/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java
deleted file mode 100644
index a3e602e719a..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldData.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.bytes;
-
-import gnu.trove.list.array.TByteArrayList;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class ByteFieldData extends NumericFieldData<ByteDocFieldData> {
-
-    static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
-
-    protected final byte[] values;
-
-    protected ByteFieldData(String fieldName, byte[] values) {
-        super(fieldName);
-        this.values = values;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return 1 * values.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    public final byte[] values() {
-        return this.values;
-    }
-
-    abstract public byte value(int docId);
-
-    abstract public byte[] values(int docId);
-
-    @Override
-    public ByteDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    protected ByteDocFieldData createFieldData() {
-        return new ByteDocFieldData(this);
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(new BytesRef(Byte.toString(values[i])));
-        }
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return new BytesRef(Byte.toString(value(docId)));
-    }
-
-    @Override
-    public byte byteValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    public short shortValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    public int intValue(int docId) {
-        return (int) value(docId);
-    }
-
-    @Override
-    public long longValue(int docId) {
-        return (long) value(docId);
-    }
-
-    @Override
-    public float floatValue(int docId) {
-        return (float) value(docId);
-    }
-
-    @Override
-    public double doubleValue(int docId) {
-        return (double) value(docId);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return FieldDataType.DefaultTypes.BYTE;
-    }
-
-    public void forEachValue(ValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(values[i]);
-        }
-    }
-
-    public static interface ValueProc {
-        void onValue(byte value);
-    }
-
-    public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
-
-    public static interface ValueInDocProc {
-        void onValue(int docId, byte value);
-
-        void onMissing(int docID);
-    }
-
-    public static ByteFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new ByteTypeLoader());
-    }
-
-    static class ByteTypeLoader extends FieldDataLoader.FreqsTypeLoader<ByteFieldData> {
-
-        private final TByteArrayList terms = new TByteArrayList();
-
-        ByteTypeLoader() {
-            super();
-            // the first one indicates null value
-            terms.add((byte) 0);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            terms.add((byte) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term));
-        }
-
-        @Override
-        public ByteFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueByteFieldData(field, ordinals, terms.toArray());
-        }
-
-        @Override
-        public ByteFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueByteFieldData(field, ordinals, terms.toArray());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataComparator.java
deleted file mode 100644
index c7abe4b95f3..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataComparator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.bytes;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Short
-public class ByteFieldDataComparator extends NumericFieldDataComparator<Byte> {
-
-    private final byte[] values;
-    private short bottom;
-
-    public ByteFieldDataComparator(int numHits, String fieldName, FieldDataCache fieldDataCache) {
-        super(fieldName, fieldDataCache);
-        values = new byte[numHits];
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.BYTE;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        return values[slot1] - values[slot2];
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        return bottom - currentFieldData.byteValue(doc);
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Byte val2) throws IOException {
-        byte val1 = currentFieldData.byteValue(doc);
-        return val1 - val2;
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        values[slot] = currentFieldData.byteValue(doc);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Byte value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java b/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java
deleted file mode 100644
index fe699897f8f..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataMissingComparator.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.bytes;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Short
-public class ByteFieldDataMissingComparator extends NumericFieldDataComparator<Byte> {
-
-    private final byte[] values;
-    private short bottom;
-    private final byte missingValue;
-
-    public ByteFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, byte missingValue) {
-        super(fieldName, fieldDataCache);
-        values = new byte[numHits];
-        this.missingValue = missingValue;
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.BYTE;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        return values[slot1] - values[slot2];
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        byte value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.byteValue(doc);
-        }
-        return bottom - value;
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Byte val2) throws IOException {
-        byte val1 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            val1 = currentFieldData.byteValue(doc);
-        }
-        return val1 - val2;
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        byte value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.byteValue(doc);
-        }
-        values[slot] = value;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Byte value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java
deleted file mode 100644
index 40201d1116e..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/bytes/ByteFieldDataType.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.bytes;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class ByteFieldDataType implements FieldDataType<ByteFieldData> {
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        if (missing == null) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new ByteFieldDataComparator(numHits, fieldname, cache);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.BYTE;
-                }
-            };
-        }
-        if (missing.equals("_last")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new ByteFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Byte.MIN_VALUE : Byte.MAX_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.BYTE;
-                }
-            };
-        }
-        if (missing.equals("_first")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new ByteFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Byte.MAX_VALUE : Byte.MIN_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.BYTE;
-                }
-            };
-        }
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new ByteFieldDataMissingComparator(numHits, fieldname, cache, Byte.parseByte(missing));
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.BYTE;
-            }
-        };
-    }
-
-    @Override
-    public ByteFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return ByteFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java b/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java
deleted file mode 100644
index e699fd0bb5f..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.bytes;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class MultiValueByteFieldData extends ByteFieldData {
-
-    private static final int VALUE_CACHE_SIZE = 10;
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<byte[][]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<byte[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<byte[][]> initialValue() {
-            byte[][] value = new byte[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new byte[i];
-            }
-            return new ThreadLocals.CleanableValue<byte[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueByteFieldData(String fieldName, int[][] ordinals, byte[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, new BytesRef(Byte.toString(values[loc])));
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onOrdinal(docId, 0);
-                }
-                break;
-            }
-            proc.onOrdinal(docId, loc);
-        }
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = doublesValuesCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        for (int i = 0; i < length; i++) {
-            doubles[i] = values[ordinals[i][docId]];
-        }
-        return doubles;
-    }
-
-    @Override
-    public byte value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return values[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public byte[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return EMPTY_BYTE_ARRAY;
-        }
-        byte[] bytes;
-        if (length < VALUE_CACHE_SIZE) {
-            bytes = valuesCache.get().get()[length];
-        } else {
-            bytes = new byte[length];
-        }
-        for (int i = 0; i < length; i++) {
-            bytes[i] = values[ordinals[i][docId]];
-        }
-        return bytes;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java b/src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java
deleted file mode 100644
index e258b0b419f..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/bytes/SingleValueByteFieldData.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to Elastic Search and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Elastic Search licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.bytes;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class SingleValueByteFieldData extends ByteFieldData {
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<byte[]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<byte[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<byte[]> initialValue() {
-            return new ThreadLocals.CleanableValue<byte[]>(new byte[1]);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueByteFieldData(String fieldName, int[] ordinals, byte[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, new BytesRef(Byte.toString(values[loc])));
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public byte value(int docId) {
-        return values[ordinals[docId]];
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = doublesValuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-
-    @Override
-    public byte[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_BYTE_ARRAY;
-        }
-        byte[] ret = valuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleDocFieldData.java
deleted file mode 100644
index 07eda570f0b..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleDocFieldData.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.doubles;
-
-import org.elasticsearch.index.field.data.NumericDocFieldData;
-
-/**
- *
- */
-public class DoubleDocFieldData extends NumericDocFieldData<DoubleFieldData> {
-
-    public DoubleDocFieldData(DoubleFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public double getValue() {
-        return fieldData.value(docId);
-    }
-
-    public double[] getValues() {
-        return fieldData.values(docId);
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java b/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java
deleted file mode 100644
index 26747293d72..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldData.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.doubles;
-
-import gnu.trove.list.array.TDoubleArrayList;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class DoubleFieldData extends NumericFieldData<DoubleDocFieldData> {
-
-    public static final double[] EMPTY_DOUBLE_ARRAY = new double[0];
-
-    protected final double[] values;
-
-    protected DoubleFieldData(String fieldName, double[] values) {
-        super(fieldName);
-        this.values = values;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return RamUsage.NUM_BYTES_DOUBLE * values.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    public final double[] values() {
-        return this.values;
-    }
-
-    abstract public double value(int docId);
-
-    abstract public double[] values(int docId);
-
-    @Override
-    public DoubleDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    protected DoubleDocFieldData createFieldData() {
-        return new DoubleDocFieldData(this);
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return new BytesRef(Double.toString(value(docId)));
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(new BytesRef(Double.toString(values[i])));
-        }
-    }
-
-    @Override
-    public byte byteValue(int docId) {
-        return (byte) value(docId);
-    }
-
-    @Override
-    public short shortValue(int docId) {
-        return (short) value(docId);
-    }
-
-    @Override
-    public int intValue(int docId) {
-        return (int) value(docId);
-    }
-
-    @Override
-    public long longValue(int docId) {
-        return (long) value(docId);
-    }
-
-    @Override
-    public float floatValue(int docId) {
-        return (float) value(docId);
-    }
-
-    @Override
-    public double doubleValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return FieldDataType.DefaultTypes.DOUBLE;
-    }
-
-    public void forEachValue(ValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(values[i]);
-        }
-    }
-
-    public static interface ValueProc {
-        void onValue(double value);
-    }
-
-    public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
-
-    public static interface ValueInDocProc {
-        void onValue(int docId, double value);
-
-        void onMissing(int docId);
-    }
-
-    public static DoubleFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new DoubleTypeLoader());
-    }
-
-    static class DoubleTypeLoader extends FieldDataLoader.FreqsTypeLoader<DoubleFieldData> {
-
-        private final TDoubleArrayList terms = new TDoubleArrayList();
-
-        DoubleTypeLoader() {
-            super();
-            // the first one indicates null value
-            terms.add(0);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            terms.add(FieldCache.NUMERIC_UTILS_DOUBLE_PARSER.parseDouble(term));
-        }
-
-        @Override
-        public DoubleFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueDoubleFieldData(field, ordinals, terms.toArray());
-        }
-
-        @Override
-        public DoubleFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueDoubleFieldData(field, ordinals, terms.toArray());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataComparator.java
deleted file mode 100644
index ea6407da50d..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataComparator.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.doubles;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Double
-public class DoubleFieldDataComparator extends NumericFieldDataComparator<Double> {
-
-    private final double[] values;
-    private double bottom;
-
-    public DoubleFieldDataComparator(int numHits, String fieldName, FieldDataCache fieldDataCache) {
-        super(fieldName, fieldDataCache);
-        values = new double[numHits];
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.DOUBLE;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final double v1 = values[slot1];
-        final double v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        final double v2 = currentFieldData.doubleValue(doc);
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Double val2) throws IOException {
-        double val1 = currentFieldData.doubleValue(doc);
-        return Double.compare(val1, val2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        values[slot] = currentFieldData.doubleValue(doc);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Double value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java b/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java
deleted file mode 100644
index 425b37bef45..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataMissingComparator.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.doubles;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Double
-public class DoubleFieldDataMissingComparator extends NumericFieldDataComparator<Double> {
-
-    private final double[] values;
-    private double bottom;
-    private final double missingValue;
-
-    public DoubleFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, double missingValue) {
-        super(fieldName, fieldDataCache);
-        values = new double[numHits];
-        this.missingValue = missingValue;
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.DOUBLE;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final double v1 = values[slot1];
-        final double v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        double v2 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            v2 = currentFieldData.doubleValue(doc);
-        }
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Double val2) throws IOException {
-        double val1 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            val1 = currentFieldData.doubleValue(doc);
-        }
-        return Double.compare(val1, val2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        double value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.doubleValue(doc);
-        }
-        values[slot] = value;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Double value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java
deleted file mode 100644
index d7bd9370937..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/doubles/DoubleFieldDataType.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.doubles;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class DoubleFieldDataType implements FieldDataType<DoubleFieldData> {
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        if (missing == null) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new DoubleFieldDataComparator(numHits, fieldname, cache);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.DOUBLE;
-                }
-            };
-        }
-        if (missing.equals("_last")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new DoubleFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.DOUBLE;
-                }
-            };
-        }
-        if (missing.equals("_first")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new DoubleFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.DOUBLE;
-                }
-            };
-        }
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new DoubleFieldDataMissingComparator(numHits, fieldname, cache, Double.parseDouble(missing));
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.DOUBLE;
-            }
-        };
-    }
-
-    @Override
-    public DoubleFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return DoubleFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java b/src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java
deleted file mode 100644
index 1b3bcfd2739..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.doubles;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-
-/**
- *
- */
-public class MultiValueDoubleFieldData extends DoubleFieldData {
-
-    private static final int VALUE_CACHE_SIZE = 10;
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueDoubleFieldData(String fieldName, int[][] ordinals, double[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, new BytesRef(Double.toString(values[loc])));
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, (long) values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, (long) values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onOrdinal(docId, 0);
-                }
-                break;
-            }
-            proc.onOrdinal(docId, loc);
-        }
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        return values(docId);
-    }
-
-    @Override
-    public double value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return values[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public double[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = valuesCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        for (int i = 0; i < length; i++) {
-            doubles[i] = values[ordinals[i][docId]];
-        }
-        return doubles;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/doubles/SingleValueDoubleFieldData.java b/src/main/java/org/elasticsearch/index/field/data/doubles/SingleValueDoubleFieldData.java
deleted file mode 100644
index 829a352a641..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/doubles/SingleValueDoubleFieldData.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.doubles;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-
-/**
- *
- */
-public class SingleValueDoubleFieldData extends DoubleFieldData {
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueDoubleFieldData(String fieldName, int[] ordinals, double[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, new BytesRef(Double.toString(values[loc])));
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, (long) values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, (long) values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        return values(docId);
-    }
-
-    @Override
-    public double value(int docId) {
-        return values[ordinals[docId]];
-    }
-
-    @Override
-    public double[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = valuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/floats/FloatDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/floats/FloatDocFieldData.java
deleted file mode 100644
index ceca0f0b3a3..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/floats/FloatDocFieldData.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.floats;
-
-import org.elasticsearch.index.field.data.NumericDocFieldData;
-
-/**
- *
- */
-public class FloatDocFieldData extends NumericDocFieldData<FloatFieldData> {
-
-    public FloatDocFieldData(FloatFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public float getValue() {
-        return fieldData.value(docId);
-    }
-
-    public float[] getValues() {
-        return fieldData.values(docId);
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldData.java b/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldData.java
deleted file mode 100644
index fe4c555ab12..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldData.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.floats;
-
-import gnu.trove.list.array.TFloatArrayList;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class FloatFieldData extends NumericFieldData<FloatDocFieldData> {
-
-    static final float[] EMPTY_FLOAT_ARRAY = new float[0];
-
-    protected final float[] values;
-
-    protected FloatFieldData(String fieldName, float[] values) {
-        super(fieldName);
-        this.values = values;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return RamUsage.NUM_BYTES_FLOAT * values.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    public final float[] values() {
-        return this.values;
-    }
-
-    abstract public float value(int docId);
-
-    abstract public float[] values(int docId);
-
-    @Override
-    public FloatDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    protected FloatDocFieldData createFieldData() {
-        return new FloatDocFieldData(this);
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return new BytesRef(Float.toString(value(docId)));
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(new BytesRef(Float.toString(values[i])));
-        }
-    }
-
-    @Override
-    public byte byteValue(int docId) {
-        return (byte) value(docId);
-    }
-
-    @Override
-    public short shortValue(int docId) {
-        return (short) value(docId);
-    }
-
-    @Override
-    public int intValue(int docId) {
-        return (int) value(docId);
-    }
-
-    @Override
-    public long longValue(int docId) {
-        return (long) value(docId);
-    }
-
-    @Override
-    public float floatValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    public double doubleValue(int docId) {
-        return (double) value(docId);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return FieldDataType.DefaultTypes.FLOAT;
-    }
-
-    public void forEachValue(ValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(values[i]);
-        }
-    }
-
-    public static interface ValueProc {
-        void onValue(float value);
-    }
-
-    public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
-
-    public static interface ValueInDocProc {
-        void onValue(int docId, float value);
-
-        void onMissing(int docId);
-    }
-
-    public static FloatFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new FloatTypeLoader());
-    }
-
-    static class FloatTypeLoader extends FieldDataLoader.FreqsTypeLoader<FloatFieldData> {
-
-        private final TFloatArrayList terms = new TFloatArrayList();
-
-        FloatTypeLoader() {
-            super();
-            // the first one indicates null value
-            terms.add(0);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            terms.add(FieldCache.NUMERIC_UTILS_FLOAT_PARSER.parseFloat(term));
-        }
-
-        @Override
-        public FloatFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueFloatFieldData(field, ordinals, terms.toArray());
-        }
-
-        @Override
-        public FloatFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueFloatFieldData(field, ordinals, terms.toArray());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataComparator.java
deleted file mode 100644
index 8341043fbdb..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataComparator.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.floats;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR - Monitor against FieldComparator.Float
-public class FloatFieldDataComparator extends NumericFieldDataComparator<Float> {
-
-    private final float[] values;
-    private float bottom;
-
-    public FloatFieldDataComparator(int numHits, String fieldName, FieldDataCache fieldDataCache) {
-        super(fieldName, fieldDataCache);
-        values = new float[numHits];
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.FLOAT;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        // TODO: are there sneaky non-branch ways to compute
-        // sign of float?
-        final float v1 = values[slot1];
-        final float v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        // TODO: are there sneaky non-branch ways to compute
-        // sign of float?
-        final float v2 = currentFieldData.floatValue(doc);
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Float val2) throws IOException {
-        float val1 = currentFieldData.floatValue(doc);
-        return Float.compare(val1, val2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        values[slot] = currentFieldData.floatValue(doc);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Float value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java b/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java
deleted file mode 100644
index 540a7fbbf00..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataMissingComparator.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.floats;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR - Monitor against FieldComparator.Float
-public class FloatFieldDataMissingComparator extends NumericFieldDataComparator<Float> {
-
-    private final float[] values;
-    private float bottom;
-    private final float missingValue;
-
-    public FloatFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, float missingValue) {
-        super(fieldName, fieldDataCache);
-        values = new float[numHits];
-        this.missingValue = missingValue;
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.FLOAT;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        // TODO: are there sneaky non-branch ways to compute
-        // sign of float?
-        final float v1 = values[slot1];
-        final float v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        // TODO: are there sneaky non-branch ways to compute
-        // sign of float?
-        float v2 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            v2 = currentFieldData.floatValue(doc);
-        }
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Float val2) throws IOException {
-        float val1 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            val1 = currentFieldData.floatValue(doc);
-        }
-        return Float.compare(val1, val2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        float value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.floatValue(doc);
-        }
-        values[slot] = value;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Float value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java
deleted file mode 100644
index 037ee25f8f5..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/floats/FloatFieldDataType.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.floats;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class FloatFieldDataType implements FieldDataType<FloatFieldData> {
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        if (missing == null) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new FloatFieldDataComparator(numHits, fieldname, cache);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.FLOAT;
-                }
-            };
-        }
-        if (missing.equals("_last")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new FloatFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.FLOAT;
-                }
-            };
-        }
-        if (missing.equals("_first")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new FloatFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.FLOAT;
-                }
-            };
-        }
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new FloatFieldDataMissingComparator(numHits, fieldname, cache, Float.parseFloat(missing));
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.FLOAT;
-            }
-        };
-    }
-
-    @Override
-    public FloatFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return FloatFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java b/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java
deleted file mode 100644
index 0b7ce83d0a4..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.floats;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class MultiValueFloatFieldData extends FloatFieldData {
-
-    private static final int VALUE_CACHE_SIZE = 10;
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<float[][]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<float[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<float[][]> initialValue() {
-            float[][] value = new float[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new float[i];
-            }
-            return new ThreadLocals.CleanableValue<float[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueFloatFieldData(String fieldName, int[][] ordinals, float[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, new BytesRef(Double.toString(values[loc])));
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, (long) values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, (long) values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onOrdinal(docId, 0);
-                }
-                break;
-            }
-            proc.onOrdinal(docId, loc);
-        }
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = doublesValuesCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        for (int i = 0; i < length; i++) {
-            doubles[i] = values[ordinals[i][docId]];
-        }
-        return doubles;
-    }
-
-    @Override
-    public float value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return values[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public float[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return EMPTY_FLOAT_ARRAY;
-        }
-        float[] floats;
-        if (length < VALUE_CACHE_SIZE) {
-            floats = valuesCache.get().get()[length];
-        } else {
-            floats = new float[length];
-        }
-        for (int i = 0; i < length; i++) {
-            floats[i] = values[ordinals[i][docId]];
-        }
-        return floats;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java b/src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java
deleted file mode 100644
index 8814ba50303..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/floats/SingleValueFloatFieldData.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.floats;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class SingleValueFloatFieldData extends FloatFieldData {
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<float[]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<float[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<float[]> initialValue() {
-            return new ThreadLocals.CleanableValue<float[]>(new float[1]);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueFloatFieldData(String fieldName, int[] ordinals, float[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, new BytesRef(Float.toString(values[loc])));
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, (long) values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, (long) values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = doublesValuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-
-    @Override
-    public float value(int docId) {
-        return values[ordinals[docId]];
-    }
-
-    @Override
-    public float[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_FLOAT_ARRAY;
-        }
-        float[] ret = valuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/ints/IntDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/ints/IntDocFieldData.java
deleted file mode 100644
index 3ebdb4a6c90..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/ints/IntDocFieldData.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.ints;
-
-import org.elasticsearch.index.field.data.NumericDocFieldData;
-
-/**
- *
- */
-public class IntDocFieldData extends NumericDocFieldData<IntFieldData> {
-
-    public IntDocFieldData(IntFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public int getValue() {
-        return fieldData.value(docId);
-    }
-
-    public int[] getValues() {
-        return fieldData.values(docId);
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java b/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java
deleted file mode 100644
index 05a77687d75..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldData.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.ints;
-
-import gnu.trove.list.array.TIntArrayList;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class IntFieldData extends NumericFieldData<IntDocFieldData> {
-
-    static final int[] EMPTY_INT_ARRAY = new int[0];
-
-    protected final int[] values;
-
-    protected IntFieldData(String fieldName, int[] values) {
-        super(fieldName);
-        this.values = values;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return RamUsage.NUM_BYTES_INT * values.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    public final int[] values() {
-        return this.values;
-    }
-
-    abstract public int value(int docId);
-
-    abstract public int[] values(int docId);
-
-    @Override
-    public IntDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    protected IntDocFieldData createFieldData() {
-        return new IntDocFieldData(this);
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return new BytesRef(Integer.toString(value(docId)));
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(new BytesRef(Integer.toString(values[i])));
-        }
-    }
-
-    @Override
-    public byte byteValue(int docId) {
-        return (byte) value(docId);
-    }
-
-    @Override
-    public short shortValue(int docId) {
-        return (short) value(docId);
-    }
-
-    @Override
-    public int intValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    public long longValue(int docId) {
-        return (long) value(docId);
-    }
-
-    @Override
-    public float floatValue(int docId) {
-        return (float) value(docId);
-    }
-
-    @Override
-    public double doubleValue(int docId) {
-        return (double) value(docId);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return FieldDataType.DefaultTypes.INT;
-    }
-
-    public void forEachValue(ValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(values[i]);
-        }
-    }
-
-    public static interface ValueProc {
-        void onValue(int value);
-    }
-
-    public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
-
-    public static interface ValueInDocProc {
-        void onValue(int docId, int value);
-
-        void onMissing(int docId);
-    }
-
-    public static IntFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new IntTypeLoader());
-    }
-
-    static class IntTypeLoader extends FieldDataLoader.FreqsTypeLoader<IntFieldData> {
-
-        private final TIntArrayList terms = new TIntArrayList();
-
-        IntTypeLoader() {
-            super();
-            // the first one indicates null value
-            terms.add(0);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            terms.add(FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term));
-        }
-
-        @Override
-        public IntFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueIntFieldData(field, ordinals, terms.toArray());
-        }
-
-        @Override
-        public IntFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueIntFieldData(field, ordinals, terms.toArray());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataComparator.java
deleted file mode 100644
index 963f116ae11..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataComparator.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.ints;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR - Monitor against FieldComparator.Int
-public class IntFieldDataComparator extends NumericFieldDataComparator<Integer> {
-
-    private final int[] values;
-
-    private int bottom;                           // Value of bottom of queue
-
-    public IntFieldDataComparator(int numHits, String fieldName, FieldDataCache fieldDataCache) {
-        super(fieldName, fieldDataCache);
-        values = new int[numHits];
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.INT;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-        // Cannot return values[slot1] - values[slot2] because that
-        // may overflow
-        final int v1 = values[slot1];
-        final int v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-        // Cannot return bottom - values[slot2] because that
-        // may overflow
-//        final int v2 = currentReaderValues[doc];
-        final int v2 = currentFieldData.intValue(doc);
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Integer val2) throws IOException {
-        int val1 = currentFieldData.intValue(doc);
-        return val1 - val2;
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        values[slot] = currentFieldData.intValue(doc);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Integer value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java b/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java
deleted file mode 100644
index a96cdac5aa1..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataMissingComparator.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.ints;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR - Monitor against FieldComparator.Int
-public class IntFieldDataMissingComparator extends NumericFieldDataComparator<Integer> {
-
-    private final int[] values;
-
-    private int bottom;                           // Value of bottom of queue
-    private final int missingValue;
-
-    public IntFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, int missingValue) {
-        super(fieldName, fieldDataCache);
-        values = new int[numHits];
-        this.missingValue = missingValue;
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.INT;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-        // Cannot return values[slot1] - values[slot2] because that
-        // may overflow
-        final int v1 = values[slot1];
-        final int v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-        // Cannot return bottom - values[slot2] because that
-        // may overflow
-//        final int v2 = currentReaderValues[doc];
-        int v2 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            v2 = currentFieldData.intValue(doc);
-        }
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Integer val2) throws IOException {
-        int val1 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            val1 = currentFieldData.intValue(doc);
-        }
-        return val1 - val2;
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        int value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.intValue(doc);
-        }
-        values[slot] = value;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Integer value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java
deleted file mode 100644
index 4b8adc65b99..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/ints/IntFieldDataType.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.ints;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class IntFieldDataType implements FieldDataType<IntFieldData> {
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        if (missing == null) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new IntFieldDataComparator(numHits, fieldname, cache);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.INT;
-                }
-            };
-        }
-        if (missing.equals("_last")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new IntFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Integer.MIN_VALUE : Integer.MAX_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.INT;
-                }
-            };
-        }
-        if (missing.equals("_first")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new IntFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Integer.MAX_VALUE : Integer.MIN_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.INT;
-                }
-            };
-        }
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new IntFieldDataMissingComparator(numHits, fieldname, cache, Integer.parseInt(missing));
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.INT;
-            }
-        };
-    }
-
-    @Override
-    public IntFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return IntFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java b/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java
deleted file mode 100644
index 26f825b12cf..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.ints;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class MultiValueIntFieldData extends IntFieldData {
-
-    private static final int VALUE_CACHE_SIZE = 10;
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<int[][]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<int[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<int[][]> initialValue() {
-            int[][] value = new int[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new int[i];
-            }
-            return new ThreadLocals.CleanableValue<int[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueIntFieldData(String fieldName, int[][] ordinals, int[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, new BytesRef(Integer.toString(values[loc])));
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onOrdinal(docId, 0);
-                }
-                break;
-            }
-            proc.onOrdinal(docId, loc);
-        }
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = doublesValuesCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        for (int i = 0; i < length; i++) {
-            doubles[i] = values[ordinals[i][docId]];
-        }
-        return doubles;
-    }
-
-    @Override
-    public int value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return values[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public int[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return EMPTY_INT_ARRAY;
-        }
-        int[] ints;
-        if (length < VALUE_CACHE_SIZE) {
-            ints = valuesCache.get().get()[length];
-        } else {
-            ints = new int[length];
-        }
-        for (int i = 0; i < length; i++) {
-            ints[i] = values[ordinals[i][docId]];
-        }
-        return ints;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java b/src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java
deleted file mode 100644
index 499066b6935..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/ints/SingleValueIntFieldData.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.ints;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class SingleValueIntFieldData extends IntFieldData {
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<int[]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<int[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<int[]> initialValue() {
-            return new ThreadLocals.CleanableValue<int[]>(new int[1]);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueIntFieldData(String fieldName, int[] ordinals, int[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, new BytesRef(Integer.toString(values[loc])));
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = doublesValuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-
-    @Override
-    public int value(int docId) {
-        return values[ordinals[docId]];
-    }
-
-    @Override
-    public int[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_INT_ARRAY;
-        }
-        int[] ret = valuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/longs/LongDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/longs/LongDocFieldData.java
deleted file mode 100644
index b16788052f3..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/longs/LongDocFieldData.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.longs;
-
-import org.elasticsearch.index.field.data.NumericDocFieldData;
-import org.joda.time.MutableDateTime;
-
-/**
- *
- */
-public class LongDocFieldData extends NumericDocFieldData<LongFieldData> {
-
-    public LongDocFieldData(LongFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public long getValue() {
-        return fieldData.value(docId);
-    }
-
-    public long[] getValues() {
-        return fieldData.values(docId);
-    }
-
-    public MutableDateTime getDate() {
-        return fieldData.date(docId);
-    }
-
-    public MutableDateTime[] getDates() {
-        return fieldData.dates(docId);
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java b/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java
deleted file mode 100644
index 736d90c6448..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldData.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.longs;
-
-import gnu.trove.list.array.TLongArrayList;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-import org.joda.time.DateTimeZone;
-import org.joda.time.MutableDateTime;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class LongFieldData extends NumericFieldData<LongDocFieldData> {
-
-    static final long[] EMPTY_LONG_ARRAY = new long[0];
-    static final MutableDateTime[] EMPTY_DATETIME_ARRAY = new MutableDateTime[0];
-
-    ThreadLocal<ThreadLocals.CleanableValue<MutableDateTime>> dateTimeCache = new ThreadLocal<ThreadLocals.CleanableValue<MutableDateTime>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<MutableDateTime> initialValue() {
-            return new ThreadLocals.CleanableValue<MutableDateTime>(new MutableDateTime(DateTimeZone.UTC));
-        }
-    };
-
-    protected final long[] values;
-
-    protected LongFieldData(String fieldName, long[] values) {
-        super(fieldName);
-        this.values = values;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return RamUsage.NUM_BYTES_LONG * values.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    public final long[] values() {
-        return this.values;
-    }
-
-    abstract public long value(int docId);
-
-    abstract public long[] values(int docId);
-
-    public MutableDateTime date(int docId) {
-        MutableDateTime dateTime = dateTimeCache.get().get();
-        dateTime.setMillis(value(docId));
-        return dateTime;
-    }
-
-    public void date(int docId, MutableDateTime dateTime) {
-        dateTime.setMillis(value(docId));
-    }
-
-    public abstract MutableDateTime[] dates(int docId);
-
-    @Override
-    public LongDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    protected LongDocFieldData createFieldData() {
-        return new LongDocFieldData(this);
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(new BytesRef(Long.toString(values[i])));
-        }
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return new BytesRef(Long.toString(value(docId)));
-    }
-
-    @Override
-    public byte byteValue(int docId) {
-        return (byte) value(docId);
-    }
-
-    @Override
-    public short shortValue(int docId) {
-        return (short) value(docId);
-    }
-
-    @Override
-    public int intValue(int docId) {
-        return (int) value(docId);
-    }
-
-    @Override
-    public long longValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    public float floatValue(int docId) {
-        return (float) value(docId);
-    }
-
-    @Override
-    public double doubleValue(int docId) {
-        return (double) value(docId);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return FieldDataType.DefaultTypes.LONG;
-    }
-
-    public void forEachValue(ValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(values[i]);
-        }
-    }
-
-    public static interface ValueProc {
-        void onValue(long value);
-    }
-
-    public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
-
-    public static interface ValueInDocProc {
-        void onValue(int docId, long value);
-
-        void onMissing(int docId);
-    }
-
-    public abstract void forEachValueInDoc(int docId, DateValueInDocProc proc);
-
-    public abstract void forEachValueInDoc(int docId, MutableDateTime dateTime, DateValueInDocProc proc);
-
-    public static interface DateValueInDocProc {
-        void onValue(int docId, MutableDateTime dateTime);
-    }
-
-    public static LongFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new LongTypeLoader());
-    }
-
-    static class LongTypeLoader extends FieldDataLoader.FreqsTypeLoader<LongFieldData> {
-
-        private final TLongArrayList terms = new TLongArrayList();
-
-        LongTypeLoader() {
-            super();
-            // the first one indicates null value
-            terms.add(0);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            terms.add(FieldCache.NUMERIC_UTILS_LONG_PARSER.parseLong(term));
-        }
-
-        @Override
-        public LongFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueLongFieldData(field, ordinals, terms.toArray());
-        }
-
-        @Override
-        public LongFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueLongFieldData(field, ordinals, terms.toArray());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataComparator.java
deleted file mode 100644
index 6de15ea4033..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataComparator.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.longs;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR - Monitor against FieldComparator.Long
-public class LongFieldDataComparator extends NumericFieldDataComparator<Long> {
-
-    private final long[] values;
-    private long bottom;
-
-    public LongFieldDataComparator(int numHits, String fieldName, FieldDataCache fieldDataCache) {
-        super(fieldName, fieldDataCache);
-        values = new long[numHits];
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.LONG;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-        final long v1 = values[slot1];
-        final long v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-//        final long v2 = currentReaderValues[doc];
-        final long v2 = currentFieldData.longValue(doc);
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Long val2) throws IOException {
-        long val1 = currentFieldData.longValue(doc);
-        return (int) (val1 - val2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        values[slot] = currentFieldData.longValue(doc);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Long value(int slot) {
-        return values[slot];
-    }
-
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java b/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java
deleted file mode 100644
index fb2617d48f2..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataMissingComparator.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.longs;
-
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR - Monitor against FieldComparator.Long
-public class LongFieldDataMissingComparator extends NumericFieldDataComparator<Long> {
-
-    private final long[] values;
-    private long bottom;
-    private final long missingValue;
-
-    public LongFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, long missingValue) {
-        super(fieldName, fieldDataCache);
-        values = new long[numHits];
-        this.missingValue = missingValue;
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.LONG;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-        final long v1 = values[slot1];
-        final long v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        // TODO: there are sneaky non-branch ways to compute
-        // -1/+1/0 sign
-//        final long v2 = currentReaderValues[doc];
-        long v2 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            v2 = currentFieldData.longValue(doc);
-        }
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Long val2) throws IOException {
-        long val1 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            val1 = currentFieldData.longValue(doc);
-        }
-        return (int) (val1 - val2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        long value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.longValue(doc);
-        }
-        values[slot] = value;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Long value(int slot) {
-        return Long.valueOf(values[slot]);
-    }
-
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java
deleted file mode 100644
index 545ad883fba..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/longs/LongFieldDataType.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.longs;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class LongFieldDataType implements FieldDataType<LongFieldData> {
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        if (missing == null) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new LongFieldDataComparator(numHits, fieldname, cache);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.LONG;
-                }
-            };
-        }
-        if (missing.equals("_last")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new LongFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Long.MIN_VALUE : Long.MAX_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.LONG;
-                }
-            };
-        }
-        if (missing.equals("_first")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new LongFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Long.MAX_VALUE : Long.MIN_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.LONG;
-                }
-            };
-        }
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new LongFieldDataMissingComparator(numHits, fieldname, cache, Long.parseLong(missing));
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.LONG;
-            }
-        };
-    }
-
-    @Override
-    public LongFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return LongFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java b/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java
deleted file mode 100644
index edaabc8f1e3..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.longs;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.joda.time.DateTimeZone;
-import org.joda.time.MutableDateTime;
-
-/**
- *
- */
-public class MultiValueLongFieldData extends LongFieldData {
-
-    private static final int VALUE_CACHE_SIZE = 10;
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<MutableDateTime[][]>> dateTimesCache = new ThreadLocal<ThreadLocals.CleanableValue<MutableDateTime[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<MutableDateTime[][]> initialValue() {
-            MutableDateTime[][] value = new MutableDateTime[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new MutableDateTime[i];
-                for (int j = 0; j < i; j++) {
-                    value[i][j] = new MutableDateTime(DateTimeZone.UTC);
-                }
-            }
-            return new ThreadLocals.CleanableValue<MutableDateTime[][]>(value);
-        }
-    };
-
-
-    private ThreadLocal<ThreadLocals.CleanableValue<long[][]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<long[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<long[][]> initialValue() {
-            long[][] value = new long[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new long[i];
-            }
-            return new ThreadLocals.CleanableValue<long[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueLongFieldData(String fieldName, int[][] ordinals, long[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, new BytesRef(Long.toString(values[loc])));
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onOrdinal(docId, 0);
-                }
-                break;
-            }
-            proc.onOrdinal(docId, loc);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DateValueInDocProc proc) {
-        MutableDateTime dateTime = dateTimeCache.get().get();
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            dateTime.setMillis(values[loc]);
-            proc.onValue(docId, dateTime);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MutableDateTime dateTime, DateValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            dateTime.setMillis(values[loc]);
-            proc.onValue(docId, dateTime);
-        }
-    }
-
-    @Override
-    public MutableDateTime[] dates(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                length++;
-            }
-        }
-        if (length == 0) {
-            return EMPTY_DATETIME_ARRAY;
-        }
-        MutableDateTime[] dates;
-        if (length < VALUE_CACHE_SIZE) {
-            dates = dateTimesCache.get().get()[length];
-        } else {
-            dates = new MutableDateTime[length];
-            for (int i = 0; i < dates.length; i++) {
-                dates[i] = new MutableDateTime();
-            }
-        }
-        int i = 0;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                dates[i++].setMillis(values[loc]);
-            }
-        }
-        return dates;
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = doublesValuesCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        for (int i = 0; i < length; i++) {
-            doubles[i] = values[ordinals[i][docId]];
-        }
-        return doubles;
-    }
-
-    @Override
-    public long value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return values[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public long[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return EMPTY_LONG_ARRAY;
-        }
-        long[] longs;
-        if (length < VALUE_CACHE_SIZE) {
-            longs = valuesCache.get().get()[length];
-        } else {
-            longs = new long[length];
-        }
-        for (int i = 0; i < length; i++) {
-            longs[i] = values[ordinals[i][docId]];
-        }
-        return longs;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java b/src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java
deleted file mode 100644
index 29561e9ed20..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/longs/SingleValueLongFieldData.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.longs;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.joda.time.DateTimeZone;
-import org.joda.time.MutableDateTime;
-
-/**
- *
- */
-public class SingleValueLongFieldData extends LongFieldData {
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<MutableDateTime[]>> datesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<MutableDateTime[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<MutableDateTime[]> initialValue() {
-            MutableDateTime[] date = new MutableDateTime[1];
-            date[0] = new MutableDateTime(DateTimeZone.UTC);
-            return new ThreadLocals.CleanableValue<MutableDateTime[]>(date);
-        }
-    };
-
-    private ThreadLocal<long[]> valuesCache = new ThreadLocal<long[]>() {
-        @Override
-        protected long[] initialValue() {
-            return new long[1];
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueLongFieldData(String fieldName, int[] ordinals, long[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, new BytesRef(Long.toString(values[loc])));
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DateValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        MutableDateTime dateTime = dateTimeCache.get().get();
-        dateTime.setMillis(values[loc]);
-        proc.onValue(docId, dateTime);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MutableDateTime dateTime, DateValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        dateTime.setMillis(values[loc]);
-        proc.onValue(docId, dateTime);
-    }
-
-    @Override
-    public MutableDateTime[] dates(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_DATETIME_ARRAY;
-        }
-        MutableDateTime[] ret = datesValuesCache.get().get();
-        ret[0].setMillis(values[loc]);
-        return ret;
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = doublesValuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-
-    @Override
-    public long value(int docId) {
-        return values[ordinals[docId]];
-    }
-
-    @Override
-    public long[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_LONG_ARRAY;
-        }
-        long[] ret = valuesCache.get();
-        ret[0] = values[loc];
-        return ret;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java b/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java
deleted file mode 100644
index 12ee6191614..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.shorts;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class MultiValueShortFieldData extends ShortFieldData {
-
-    private static final int VALUE_CACHE_SIZE = 10;
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<short[][]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<short[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<short[][]> initialValue() {
-            short[][] value = new short[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new short[i];
-            }
-            return new ThreadLocals.CleanableValue<short[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueShortFieldData(String fieldName, int[][] ordinals, short[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        boolean found = false;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                found = true;
-                proc.onValue(docId, new BytesRef(Short.toString(values[loc])));
-            }
-        }
-        if (!found) {
-            proc.onMissing(docId);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                proc.onValue(docId, values[loc]);
-            }
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                proc.onValue(docId, values[loc]);
-            }
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        boolean found = false;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                found = true;
-                proc.onValue(docId, values[loc]);
-            }
-        }
-        if (!found) {
-            proc.onMissing(docId);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        boolean found = false;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                found = true;
-                proc.onValue(docId, values[loc]);
-            }
-        }
-        if (!found) {
-            proc.onMissing(docId);
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        boolean found = false;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                found = true;
-                proc.onValue(docId, values[loc]);
-            }
-        }
-        if (!found) {
-            proc.onMissing(docId);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        boolean found = false;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                found = true;
-                proc.onOrdinal(docId, loc);
-            }
-        }
-        if (!found) {
-            proc.onOrdinal(docId, 0);
-        }
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                length++;
-            }
-        }
-        if (length == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = doublesValuesCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        int i = 0;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                doubles[i++] = values[loc];
-            }
-        }
-        return doubles;
-    }
-
-    @Override
-    public short value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return values[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public short[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                length++;
-            }
-        }
-        if (length == 0) {
-            return EMPTY_SHORT_ARRAY;
-        }
-        short[] shorts;
-        if (length < VALUE_CACHE_SIZE) {
-            shorts = valuesCache.get().get()[length];
-        } else {
-            shorts = new short[length];
-        }
-        int i = 0;
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                shorts[i++] = values[loc];
-            }
-        }
-        return shorts;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/shorts/ShortDocFieldData.java
deleted file mode 100644
index 014dc7eb677..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortDocFieldData.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.shorts;
-
-import org.elasticsearch.index.field.data.NumericDocFieldData;
-
-/**
- *
- */
-public class ShortDocFieldData extends NumericDocFieldData<ShortFieldData> {
-
-    public ShortDocFieldData(ShortFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public short getValue() {
-        return fieldData.value(docId);
-    }
-
-    public short[] getValues() {
-        return fieldData.values(docId);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java b/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java
deleted file mode 100644
index 3c02442255a..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldData.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.shorts;
-
-import gnu.trove.list.array.TShortArrayList;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class ShortFieldData extends NumericFieldData<ShortDocFieldData> {
-
-    static final short[] EMPTY_SHORT_ARRAY = new short[0];
-
-    protected final short[] values;
-
-    protected ShortFieldData(String fieldName, short[] values) {
-        super(fieldName);
-        this.values = values;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return RamUsage.NUM_BYTES_SHORT * values.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    public final short[] values() {
-        return this.values;
-    }
-
-    abstract public short value(int docId);
-
-    abstract public short[] values(int docId);
-
-    @Override
-    public ShortDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    protected ShortDocFieldData createFieldData() {
-        return new ShortDocFieldData(this);
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(new BytesRef(Short.toString(values[i])));
-        }
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return new BytesRef(Short.toString(value(docId)));
-    }
-
-    @Override
-    public byte byteValue(int docId) {
-        return (byte) value(docId);
-    }
-
-    @Override
-    public short shortValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    public int intValue(int docId) {
-        return (int) value(docId);
-    }
-
-    @Override
-    public long longValue(int docId) {
-        return (long) value(docId);
-    }
-
-    @Override
-    public float floatValue(int docId) {
-        return (float) value(docId);
-    }
-
-    @Override
-    public double doubleValue(int docId) {
-        return (double) value(docId);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return FieldDataType.DefaultTypes.SHORT;
-    }
-
-    public void forEachValue(ValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(values[i]);
-        }
-    }
-
-    public static interface ValueProc {
-        void onValue(short value);
-    }
-
-    public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
-
-    public static interface ValueInDocProc {
-        void onValue(int docId, short value);
-
-        void onMissing(int docId);
-    }
-
-    public static ShortFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new ShortTypeLoader());
-    }
-
-    static class ShortTypeLoader extends FieldDataLoader.FreqsTypeLoader<ShortFieldData> {
-
-        private final TShortArrayList terms = new TShortArrayList();
-
-        ShortTypeLoader() {
-            super();
-            // the first one indicates null value
-            terms.add((short) 0);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            terms.add((short) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term));
-        }
-
-        @Override
-        public ShortFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueShortFieldData(field, ordinals, terms.toArray());
-        }
-
-        @Override
-        public ShortFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueShortFieldData(field, ordinals, terms.toArray());
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataComparator.java
deleted file mode 100644
index 0d437899625..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataComparator.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.shorts;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Short
-public class ShortFieldDataComparator extends NumericFieldDataComparator<Short> {
-
-    private final short[] values;
-    private short bottom;
-
-    public ShortFieldDataComparator(int numHits, String fieldName, FieldDataCache fieldDataCache) {
-        super(fieldName, fieldDataCache);
-        values = new short[numHits];
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.SHORT;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        return values[slot1] - values[slot2];
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        return bottom - currentFieldData.shortValue(doc);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        values[slot] = currentFieldData.shortValue(doc);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Short val2) throws IOException {
-        short val1 = currentFieldData.shortValue(doc);
-        return val1 - val2;
-    }
-
-    @Override
-    public Short value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java b/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java
deleted file mode 100644
index a85e3510217..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataMissingComparator.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.shorts;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.NumericFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Short
-public class ShortFieldDataMissingComparator extends NumericFieldDataComparator<Short> {
-
-    private final short[] values;
-    private short bottom;
-    private final short missingValue;
-
-    public ShortFieldDataMissingComparator(int numHits, String fieldName, FieldDataCache fieldDataCache, short missingValue) {
-        super(fieldName, fieldDataCache);
-        values = new short[numHits];
-        this.missingValue = missingValue;
-    }
-
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.SHORT;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        return values[slot1] - values[slot2];
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        short value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.shortValue(doc);
-        }
-        return bottom - value;
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        short value = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            value = currentFieldData.shortValue(doc);
-        }
-        values[slot] = value;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Short val2) throws IOException {
-        short val1 = missingValue;
-        if (currentFieldData.hasValue(doc)) {
-            val1 = currentFieldData.shortValue(doc);
-        }
-        return val1 - val2;
-    }
-
-    @Override
-    public Short value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java
deleted file mode 100644
index b69d47ef031..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/shorts/ShortFieldDataType.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.shorts;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class ShortFieldDataType implements FieldDataType<ShortFieldData> {
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        if (missing == null) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new ShortFieldDataComparator(numHits, fieldname, cache);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.SHORT;
-                }
-            };
-        }
-        if (missing.equals("_last")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new ShortFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Short.MIN_VALUE : Short.MAX_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.SHORT;
-                }
-            };
-        }
-        if (missing.equals("_first")) {
-            return new ExtendedFieldComparatorSource() {
-                @Override
-                public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                    return new ShortFieldDataMissingComparator(numHits, fieldname, cache, reversed ? Short.MAX_VALUE : Short.MIN_VALUE);
-                }
-
-                @Override
-                public SortField.Type reducedType() {
-                    return SortField.Type.SHORT;
-                }
-            };
-        }
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new ShortFieldDataMissingComparator(numHits, fieldname, cache, Short.parseShort(missing));
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.SHORT;
-            }
-        };
-    }
-
-    @Override
-    public ShortFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return ShortFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java b/src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java
deleted file mode 100644
index 732a0cd81ec..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/shorts/SingleValueShortFieldData.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.shorts;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-
-/**
- *
- */
-public class SingleValueShortFieldData extends ShortFieldData {
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> doublesValuesCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<short[]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<short[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<short[]> initialValue() {
-            return new ThreadLocals.CleanableValue<short[]>(new short[1]);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueShortFieldData(String fieldName, int[] ordinals, short[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, new BytesRef(Short.toString(values[loc])));
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, DoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, LongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingDoubleValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, MissingLongValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public short value(int docId) {
-        return values[ordinals[docId]];
-    }
-
-    @Override
-    public double[] doubleValues(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = doublesValuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-
-    @Override
-    public short[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_SHORT_ARRAY;
-        }
-        short[] ret = valuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java b/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java
deleted file mode 100644
index 1cbf77876ea..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.strings;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-
-/**
- *
- */
-public class MultiValueStringFieldData extends StringFieldData {
-
-    private static final BytesRef[] EMPTY_ARRAY = new BytesRef[0];
-
-    private static final int VALUE_CACHE_SIZE = 100;
-
-    private static ThreadLocal<ThreadLocals.CleanableValue<BytesRef[][]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<BytesRef[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<BytesRef[][]> initialValue() {
-            BytesRef[][] value = new BytesRef[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new BytesRef[i];
-            }
-            return new ThreadLocals.CleanableValue<BytesRef[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueStringFieldData(String fieldName, int[][] ordinals, BytesRef[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            proc.onValue(docId, values[loc]);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onOrdinal(docId, 0);
-                }
-                break;
-            }
-            proc.onOrdinal(docId, loc);
-        }
-    }
-
-    @Override
-    public BytesRef value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return values[loc];
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public BytesRef[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return EMPTY_ARRAY;
-        }
-        BytesRef[] refs;
-        if (length < VALUE_CACHE_SIZE) {
-            refs = valuesCache.get().get()[length];
-        } else {
-            refs = new BytesRef[length];
-        }
-        for (int i = 0; i < length; i++) {
-            refs[i] = values[ordinals[i][docId]];
-        }
-        return refs;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java b/src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java
deleted file mode 100644
index b13c6cf4dfd..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/strings/SingleValueStringFieldData.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.strings;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-
-/**
- *
- */
-public class SingleValueStringFieldData extends StringFieldData {
-
-    private static final BytesRef[] EMPTY_ARRAY = new BytesRef[0];
-
-    private static ThreadLocal<ThreadLocals.CleanableValue<BytesRef[]>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<BytesRef[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<BytesRef[]> initialValue() {
-            return new ThreadLocals.CleanableValue<BytesRef[]>(new BytesRef[1]);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueStringFieldData(String fieldName, int[] ordinals, BytesRef[] values) {
-        super(fieldName, values);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    int[] ordinals() {
-        return ordinals;
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        proc.onValue(docId, values[loc]);
-    }
-
-    @Override
-    public BytesRef value(int docId) {
-        return values[ordinals[docId]];
-    }
-
-    @Override
-    public BytesRef[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_ARRAY;
-        }
-        BytesRef[] ret = valuesCache.get().get();
-        ret[0] = values[loc];
-        return ret;
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/strings/StringDocFieldData.java b/src/main/java/org/elasticsearch/index/field/data/strings/StringDocFieldData.java
deleted file mode 100644
index 2c75ed6d63e..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/strings/StringDocFieldData.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.strings;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.field.data.DocFieldData;
-
-/**
- *
- */
-public class StringDocFieldData extends DocFieldData<StringFieldData> {
-
-    public StringDocFieldData(StringFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public String getValue() {
-        BytesRef value = fieldData.value(docId);
-        if (value == null) {
-            return null;
-        }
-        return value.utf8ToString();
-    }
-
-    public String[] getValues() {
-        BytesRef[] values = fieldData.values(docId);
-        if (values == null) {
-            return null;
-        }
-        String[] stringValues = new String[values.length];
-        for (int i = 0; i < values.length; i++) {
-            stringValues[i] = values[i].utf8ToString();
-        }
-        return stringValues;
-    }
-
-    public BytesRef getBytesValue() {
-        return fieldData.value(docId);
-    }
-
-    public BytesRef[] getBytesValues() {
-        return fieldData.values(docId);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldData.java b/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldData.java
deleted file mode 100644
index 1f45f5d1fa4..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldData.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.strings;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-/**
- *
- */
-public abstract class StringFieldData extends FieldData<StringDocFieldData> {
-
-    protected final BytesRef[] values;
-
-    protected StringFieldData(String fieldName, BytesRef[] values) {
-        super(fieldName);
-        this.values = values;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = RamUsage.NUM_BYTES_ARRAY_HEADER;
-        for (BytesRef value : values) {
-            if (value != null) {
-                size += RamUsage.NUM_BYTES_OBJECT_REF + RamUsage.NUM_BYTES_OBJECT_HEADER +
-                        RamUsage.NUM_BYTES_ARRAY_HEADER + (value.length + (2 * RamUsage.NUM_BYTES_INT));
-            }
-        }
-        return size;
-    }
-
-    public BytesRef[] values() {
-        return this.values;
-    }
-
-    abstract public BytesRef value(int docId);
-
-    abstract public BytesRef[] values(int docId);
-
-    @Override
-    public StringDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return value(docId);
-    }
-
-    @Override
-    protected StringDocFieldData createFieldData() {
-        return new StringDocFieldData(this);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return FieldDataType.DefaultTypes.STRING;
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < values.length; i++) {
-            proc.onValue(values[i]);
-        }
-    }
-
-    public static StringFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new StringTypeLoader());
-    }
-
-    static class StringTypeLoader extends FieldDataLoader.FreqsTypeLoader<StringFieldData> {
-
-        private final ArrayList<BytesRef> terms = new ArrayList<BytesRef>();
-
-        StringTypeLoader() {
-            super();
-            // the first one indicates null value
-            terms.add(null);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            terms.add(term);
-        }
-
-        @Override
-        public StringFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueStringFieldData(field, ordinals, terms.toArray(new BytesRef[terms.size()]));
-        }
-
-        @Override
-        public StringFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueStringFieldData(field, ordinals, terms.toArray(new BytesRef[terms.size()]));
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java b/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java
deleted file mode 100644
index c5e9f797850..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/strings/StringFieldDataType.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.strings;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class StringFieldDataType implements FieldDataType<StringFieldData> {
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        if (missing != null) {
-            throw new ElasticSearchIllegalArgumentException("Sorting on string type field does not support missing parameter");
-        }
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new StringOrdValFieldDataComparator(numHits, fieldname, sortPos, reversed, cache);
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.STRING;
-            }
-        };
-    }
-
-    @Override
-    public StringFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return StringFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/strings/StringOrdValFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/strings/StringOrdValFieldDataComparator.java
deleted file mode 100644
index 0c9d6dd6d6a..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/strings/StringOrdValFieldDataComparator.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.strings;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator#String
-public class StringOrdValFieldDataComparator extends FieldComparator<BytesRef> {
-
-    private final FieldDataCache fieldDataCache;
-
-    private final int[] ords;
-    private final BytesRef[] values;
-    private final int[] readerGen;
-
-    private int currentReaderGen = -1;
-    private BytesRef[] lookup;
-    private int[] order;
-    private final String field;
-
-    private int bottomSlot = -1;
-    private int bottomOrd;
-    private boolean bottomSameReader;
-    private BytesRef bottomValue;
-
-    public StringOrdValFieldDataComparator(int numHits, String field, int sortPos, boolean reversed, FieldDataCache fieldDataCache) {
-        this.fieldDataCache = fieldDataCache;
-        ords = new int[numHits];
-        values = new BytesRef[numHits];
-        readerGen = new int[numHits];
-        this.field = field;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        if (readerGen[slot1] == readerGen[slot2]) {
-            return ords[slot1] - ords[slot2];
-        }
-
-        final BytesRef val1 = values[slot1];
-        final BytesRef val2 = values[slot2];
-        if (val1 == null) {
-            if (val2 == null) {
-                return 0;
-            }
-            return -1;
-        } else if (val2 == null) {
-            return 1;
-        }
-        return val1.compareTo(val2);
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        assert bottomSlot != -1;
-        if (bottomSameReader) {
-            // ord is precisely comparable, even in the equal case
-            return bottomOrd - this.order[doc];
-        } else {
-            // ord is only approx comparable: if they are not
-            // equal, we can use that; if they are equal, we
-            // must fallback to compare by value
-            final int order = this.order[doc];
-            final int cmp = bottomOrd - order;
-            if (cmp != 0) {
-                return cmp;
-            }
-
-            final BytesRef val2 = lookup[order];
-            if (bottomValue == null) {
-                if (val2 == null) {
-                    return 0;
-                }
-                // bottom wins
-                return -1;
-            } else if (val2 == null) {
-                // doc wins
-                return 1;
-            }
-            return bottomValue.compareTo(val2);
-        }
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        final int ord = order[doc];
-        ords[slot] = ord;
-        assert ord >= 0;
-        values[slot] = lookup[ord];
-        readerGen[slot] = currentReaderGen;
-    }
-
-    @Override
-    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-        FieldData cleanFieldData = fieldDataCache.cache(FieldDataType.DefaultTypes.STRING, context.reader(), field);
-        if (cleanFieldData instanceof MultiValueStringFieldData) {
-            throw new IOException("Can't sort on string types with more than one value per doc, or more than one token per field");
-        }
-        SingleValueStringFieldData fieldData = (SingleValueStringFieldData) cleanFieldData;
-        currentReaderGen++;
-        order = fieldData.ordinals();
-        lookup = fieldData.values();
-        assert lookup.length > 0;
-        if (bottomSlot != -1) {
-            setBottom(bottomSlot);
-        }
-        return this;
-    }
-
-    @Override
-    public int compareDocToValue(int doc, BytesRef otherVal) throws IOException {
-        BytesRef val = values[ords[doc]];
-        if (otherVal == null) {
-            if (val == null) {
-                return 0;
-            }
-            return -1;
-        } else if (val == null) {
-            return 1;
-        }
-        return val.compareTo(otherVal);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        bottomSlot = bottom;
-
-        bottomValue = values[bottomSlot];
-        if (currentReaderGen == readerGen[bottomSlot]) {
-            bottomOrd = ords[bottomSlot];
-            bottomSameReader = true;
-        } else {
-            if (bottomValue == null) {
-                ords[bottomSlot] = 0;
-                bottomOrd = 0;
-                bottomSameReader = true;
-                readerGen[bottomSlot] = currentReaderGen;
-            } else {
-                final int index = binarySearch(bottomValue, lookup);
-                if (index < 0) {
-                    bottomOrd = -index - 2;
-                    bottomSameReader = false;
-                } else {
-                    bottomOrd = index;
-                    // exact value match
-                    bottomSameReader = true;
-                    readerGen[bottomSlot] = currentReaderGen;
-                    ords[bottomSlot] = bottomOrd;
-                }
-            }
-        }
-    }
-
-    private static int binarySearch(BytesRef value, BytesRef[] values) {
-        return binarySearch(value, values, 1, values.length-1);
-    }
-
-    private static int binarySearch(BytesRef value, BytesRef[] values, int low, int high) {
-        while (low <= high) {
-            int mid = (low + high) >>> 1;
-            BytesRef midVal = values[mid];
-            int cmp;
-            if (midVal != null) {
-                cmp = midVal.compareTo(value);
-            } else {
-                cmp = -1;
-            }
-
-            if (cmp < 0)
-                low = mid + 1;
-            else if (cmp > 0)
-                high = mid - 1;
-            else
-                return mid;
-        }
-        return -(low + 1);
-    }
-
-    @Override
-    public BytesRef value(int slot) {
-        return values[slot];
-    }
-
-    public BytesRef[] getValues() {
-        return values;
-    }
-
-    public int getBottomSlot() {
-        return bottomSlot;
-    }
-
-    public String getField() {
-        return field;
-    }
-
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/strings/StringValFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/strings/StringValFieldDataComparator.java
deleted file mode 100644
index c9103a82dd5..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/strings/StringValFieldDataComparator.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.strings;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator#String
-public class StringValFieldDataComparator extends FieldComparator<BytesRef> {
-
-    private final String fieldName;
-
-    protected final FieldDataCache fieldDataCache;
-
-    protected FieldData currentFieldData;
-
-    private BytesRef[] values;
-
-    private BytesRef bottom;
-
-    public StringValFieldDataComparator(int numHits, String fieldName, FieldDataCache fieldDataCache) {
-        this.fieldName = fieldName;
-        this.fieldDataCache = fieldDataCache;
-        values = new BytesRef[numHits];
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final BytesRef val1 = values[slot1];
-        final BytesRef val2 = values[slot2];
-        if (val1 == null) {
-            if (val2 == null) {
-                return 0;
-            }
-            return -1;
-        } else if (val2 == null) {
-            return 1;
-        }
-
-        return val1.compareTo(val2);
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        final BytesRef val2 = currentFieldData.stringValue(doc);
-        if (bottom == null) {
-            if (val2 == null) {
-                return 0;
-            }
-            return -1;
-        } else if (val2 == null) {
-            return 1;
-        }
-        return bottom.compareTo(val2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        values[slot] = currentFieldData.stringValue(doc);
-    }
-
-    @Override
-    public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-        currentFieldData = fieldDataCache.cache(FieldDataType.DefaultTypes.STRING, context.reader(), fieldName);
-        return this;
-    }
-
-    @Override
-    public int compareDocToValue(int doc, BytesRef val2) throws IOException {
-        BytesRef val1 = currentFieldData.stringValue(doc);
-        if (val1 == null) {
-            if (val2 == null) {
-                return 0;
-            }
-            return -1;
-        } else if (val2 == null) {
-            return 1;
-        }
-        return currentFieldData.stringValue(doc).compareTo(val2);
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public BytesRef value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/support/FieldDataLoader.java b/src/main/java/org/elasticsearch/index/field/data/support/FieldDataLoader.java
deleted file mode 100644
index a52525f5438..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/support/FieldDataLoader.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.support;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.field.data.FieldData;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-/**
- *
- */
-public class FieldDataLoader {
-
-    @SuppressWarnings({"StringEquality"})
-    public static <T extends FieldData> T load(AtomicReader reader, String field, TypeLoader<T> loader) throws IOException {
-
-        loader.init();
-        ArrayList<int[]> ordinals = new ArrayList<int[]>();
-        int[] idx = new int[reader.maxDoc()];
-        ordinals.add(new int[reader.maxDoc()]);
-
-        int t = 1;  // current term number
-
-        Terms terms = reader.terms(field);
-        if (terms == null) {
-            return  loader.buildSingleValue(field, ordinals.get(0)); // Return empty field data if field doesn't exists.
-        }
-
-        TermsEnum termsEnum = terms.iterator(null);
-        try {
-            DocsEnum docsEnum = null;
-            for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
-                loader.collectTerm(BytesRef.deepCopyOf(term));
-                docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
-                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
-                    int[] ordinal;
-                    if (idx[docId] >= ordinals.size()) {
-                        ordinal = new int[reader.maxDoc()];
-                        ordinals.add(ordinal);
-                    } else {
-                        ordinal = ordinals.get(idx[docId]);
-                    }
-                    ordinal[docId] = t;
-                    idx[docId]++;
-                }
-                t++;
-            }
-        } catch (RuntimeException e) {
-            if (e.getClass().getName().endsWith("StopFillCacheException")) {
-                // all is well, in case numeric parsers are used.
-            } else {
-                throw e;
-            }
-        }
-
-        if (ordinals.size() == 1) {
-            return loader.buildSingleValue(field, ordinals.get(0));
-        } else {
-            int[][] nativeOrdinals = new int[ordinals.size()][];
-            for (int i = 0; i < nativeOrdinals.length; i++) {
-                nativeOrdinals[i] = ordinals.get(i);
-            }
-            return loader.buildMultiValue(field, nativeOrdinals);
-        }
-    }
-
-    public static interface TypeLoader<T extends FieldData> {
-
-        void init();
-
-        void collectTerm(BytesRef term);
-
-        T buildSingleValue(String fieldName, int[] ordinals);
-
-        T buildMultiValue(String fieldName, int[][] ordinals);
-    }
-
-    public static abstract class FreqsTypeLoader<T extends FieldData> implements TypeLoader<T> {
-
-        protected FreqsTypeLoader() {
-        }
-
-        @Override
-        public void init() {
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/field/data/support/NumericFieldDataComparator.java b/src/main/java/org/elasticsearch/index/field/data/support/NumericFieldDataComparator.java
deleted file mode 100644
index f56a3bb10d2..00000000000
--- a/src/main/java/org/elasticsearch/index/field/data/support/NumericFieldDataComparator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.field.data.support;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldComparator;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.NumericFieldData;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class NumericFieldDataComparator<N extends Number> extends FieldComparator<N> {
-
-    private final String fieldName;
-
-    protected final FieldDataCache fieldDataCache;
-
-    protected NumericFieldData currentFieldData;
-
-    public NumericFieldDataComparator(String fieldName, FieldDataCache fieldDataCache) {
-        this.fieldName = fieldName;
-        this.fieldDataCache = fieldDataCache;
-    }
-
-    public abstract FieldDataType fieldDataType();
-
-    @Override
-    public NumericFieldDataComparator<N> setNextReader(AtomicReaderContext context) throws IOException {
-        currentFieldData = (NumericFieldData) fieldDataCache.cache(fieldDataType(), context.reader(), fieldName);
-        return this;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index f8fcd863c9b..1977aa531dd 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -109,7 +109,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
     }
 
     public <IFD extends IndexFieldData> IFD getForField(FieldMapper mapper) {
-        return getForField(mapper.names(), mapper.fieldDataType2());
+        return getForField(mapper.names(), mapper.fieldDataType());
     }
 
     public <IFD extends IndexFieldData> IFD getForField(FieldMapper.Names fieldNames, FieldDataType type) {
diff --git a/src/main/java/org/elasticsearch/index/field/function/sort/DoubleFieldsFunctionDataComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java
similarity index 85%
rename from src/main/java/org/elasticsearch/index/field/function/sort/DoubleFieldsFunctionDataComparator.java
rename to src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java
index d130be283e3..d9533f4ad35 100644
--- a/src/main/java/org/elasticsearch/index/field/function/sort/DoubleFieldsFunctionDataComparator.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java
@@ -17,13 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.index.field.function.sort;
+package org.elasticsearch.index.fielddata.fieldcomparator;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.script.SearchScript;
 
 import java.io.IOException;
@@ -32,13 +32,13 @@ import java.io.IOException;
  *
  */
 // LUCENE MONITOR: Monitor against FieldComparator.Double
-public class DoubleFieldsFunctionDataComparator extends FieldComparator<Double> {
+public class DoubleScriptDataComparator extends FieldComparator<Double> {
 
-    public static FieldDataType.ExtendedFieldComparatorSource comparatorSource(SearchScript script) {
+    public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) {
         return new InnerSource(script);
     }
 
-    private static class InnerSource extends FieldDataType.ExtendedFieldComparatorSource {
+    private static class InnerSource extends IndexFieldData.XFieldComparatorSource {
 
         private final SearchScript script;
 
@@ -48,7 +48,7 @@ public class DoubleFieldsFunctionDataComparator extends FieldComparator<Double>
 
         @Override
         public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-            return new DoubleFieldsFunctionDataComparator(numHits, script);
+            return new DoubleScriptDataComparator(numHits, script);
         }
 
         @Override
@@ -62,7 +62,7 @@ public class DoubleFieldsFunctionDataComparator extends FieldComparator<Double>
     private final double[] values;
     private double bottom;
 
-    public DoubleFieldsFunctionDataComparator(int numHits, SearchScript script) {
+    public DoubleScriptDataComparator(int numHits, SearchScript script) {
         this.script = script;
         values = new double[numHits];
     }
diff --git a/src/main/java/org/elasticsearch/index/field/function/sort/StringFieldsFunctionDataComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java
similarity index 85%
rename from src/main/java/org/elasticsearch/index/field/function/sort/StringFieldsFunctionDataComparator.java
rename to src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java
index 58a18b6ad4f..74f301429cb 100644
--- a/src/main/java/org/elasticsearch/index/field/function/sort/StringFieldsFunctionDataComparator.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java
@@ -17,13 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.index.field.function.sort;
+package org.elasticsearch.index.fielddata.fieldcomparator;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.script.SearchScript;
 
 import java.io.IOException;
@@ -31,13 +31,13 @@ import java.io.IOException;
 /**
  *
  */
-public class StringFieldsFunctionDataComparator extends FieldComparator<String> {
+public class StringScriptDataComparator extends FieldComparator<String> {
 
-    public static FieldDataType.ExtendedFieldComparatorSource comparatorSource(SearchScript script) {
+    public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) {
         return new InnerSource(script);
     }
 
-    private static class InnerSource extends FieldDataType.ExtendedFieldComparatorSource {
+    private static class InnerSource extends IndexFieldData.XFieldComparatorSource {
 
         private final SearchScript script;
 
@@ -47,7 +47,7 @@ public class StringFieldsFunctionDataComparator extends FieldComparator<String>
 
         @Override
         public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-            return new StringFieldsFunctionDataComparator(numHits, script);
+            return new StringScriptDataComparator(numHits, script);
         }
 
         @Override
@@ -62,7 +62,7 @@ public class StringFieldsFunctionDataComparator extends FieldComparator<String>
 
     private String bottom;
 
-    public StringFieldsFunctionDataComparator(int numHits, SearchScript script) {
+    public StringScriptDataComparator(int numHits, SearchScript script) {
         this.script = script;
         values = new String[numHits];
     }
diff --git a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index e0f95d57658..e3a4ff96e8b 100644
--- a/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -28,7 +28,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 
@@ -201,7 +201,5 @@ public interface FieldMapper<T> {
 
     FieldDataType fieldDataType();
 
-    org.elasticsearch.index.fielddata.FieldDataType fieldDataType2();
-
     PostingsFormatProvider postingsFormatProvider();
 }
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
index cf616a9775e..5ee779c7824 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
@@ -37,7 +37,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingFormats;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -551,11 +550,6 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
         }
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.STRING;
-    }
-
     @Override
     public PostingsFormatProvider postingsFormatProvider() {
         return postingsFormat;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
index 01dfcde665b..b43542b801c 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
@@ -134,7 +134,7 @@ public class BinaryFieldMapper extends AbstractFieldMapper<BytesReference> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType() {
         throw new ElasticSearchIllegalArgumentException("field data on binary field is not supported");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
index 729ddb3f794..3d8a0e9515f 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
@@ -23,7 +23,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
@@ -31,6 +30,7 @@ import org.elasticsearch.common.lucene.search.TermFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
@@ -161,8 +161,9 @@ public class BooleanFieldMapper extends AbstractFieldMapper<Boolean> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        throw new ElasticSearchIllegalArgumentException("not implemented");
+    public FieldDataType fieldDataType() {
+        // TODO have a special boolean type?
+        return new FieldDataType("string");
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
index 3d75355912b..0ec5cd3f959 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java
@@ -38,7 +38,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -130,8 +130,8 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        return new org.elasticsearch.index.fielddata.FieldDataType("byte");
+    public FieldDataType fieldDataType() {
+        return new FieldDataType("byte");
     }
 
     @Override
@@ -327,11 +327,6 @@ public class ByteFieldMapper extends NumberFieldMapper<Byte> {
         return field;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.BYTE;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 0982cd6b41e..0588fcd161c 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -41,7 +41,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericDateAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -174,9 +174,8 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        // long for now, need to think about scripts and getting a DateTime back?
-        return new org.elasticsearch.index.fielddata.FieldDataType("long");
+    public FieldDataType fieldDataType() {
+        return new FieldDataType("long");
     }
 
     @Override
@@ -404,11 +403,6 @@ public class DateFieldMapper extends NumberFieldMapper<Long> {
         return field;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.LONG;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
index 4e2197471dc..97847bc6742 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java
@@ -38,7 +38,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericDoubleAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -133,8 +133,8 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        return new org.elasticsearch.index.fielddata.FieldDataType("double");
+    public FieldDataType fieldDataType() {
+        return new FieldDataType("double");
     }
 
     @Override
@@ -327,11 +327,6 @@ public class DoubleFieldMapper extends NumberFieldMapper<Double> {
         return field;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.DOUBLE;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
index bc8f4dcceb2..52f52f9129e 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java
@@ -39,7 +39,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericFloatAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -131,8 +131,8 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        return new org.elasticsearch.index.fielddata.FieldDataType("float");
+    public FieldDataType fieldDataType() {
+        return new FieldDataType("float");
     }
 
     @Override
@@ -321,11 +321,6 @@ public class FloatFieldMapper extends NumberFieldMapper<Float> {
         return field;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.FLOAT;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
index 2057e08bbd9..aaa045c0648 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java
@@ -39,7 +39,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -133,7 +132,7 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType() {
         return new org.elasticsearch.index.fielddata.FieldDataType("int");
     }
 
@@ -327,11 +326,6 @@ public class IntegerFieldMapper extends NumberFieldMapper<Integer> {
         return field;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.INT;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
index 0c1858b40a9..b1f86650636 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java
@@ -39,7 +39,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericLongAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -133,8 +133,8 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
-        return new org.elasticsearch.index.fielddata.FieldDataType("long");
+    public FieldDataType fieldDataType() {
+        return new FieldDataType("long");
     }
 
     @Override
@@ -326,11 +326,6 @@ public class LongFieldMapper extends NumberFieldMapper<Long> {
         return field;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.LONG;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index 2f44f9fc241..ca0d9415660 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -31,7 +31,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
@@ -277,9 +276,6 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
         tokenStream.remove();
     }
 
-    @Override
-    public abstract FieldDataType fieldDataType();
-
     protected NumericTokenStream popCachedStream() {
         return tokenStream.get();
     }
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
index f76209512a6..65afdf63e15 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java
@@ -39,7 +39,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericIntegerAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -133,7 +132,7 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType() {
         return new org.elasticsearch.index.fielddata.FieldDataType("short");
     }
 
@@ -330,11 +329,6 @@ public class ShortFieldMapper extends NumberFieldMapper<Short> {
         return field;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.SHORT;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
index 6e071572c82..20d89f27674 100644
--- a/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
@@ -213,7 +213,7 @@ public class StringFieldMapper extends AbstractFieldMapper<String> implements Al
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         return new FieldDataType("string");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointDocFieldData.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointDocFieldData.java
deleted file mode 100644
index f807f666022..00000000000
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointDocFieldData.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper.geo;
-
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.field.data.DocFieldData;
-
-/**
- *
- */
-public class GeoPointDocFieldData extends DocFieldData<GeoPointFieldData> {
-
-    public GeoPointDocFieldData(GeoPointFieldData fieldData) {
-        super(fieldData);
-    }
-
-    public GeoPoint getValue() {
-        return fieldData.value(docId);
-    }
-
-    public GeoPoint[] getValues() {
-        return fieldData.values(docId);
-    }
-
-    public double factorDistance(double lat, double lon) {
-        return fieldData.factorDistance(docId, DistanceUnit.MILES, lat, lon);
-    }
-
-    public double factorDistanceWithDefault(double lat, double lon, double defaultValue) {
-        if (!fieldData.hasValue(docId)) {
-            return defaultValue;
-        }
-        return fieldData.factorDistance(docId, DistanceUnit.MILES, lat, lon);
-    }
-
-    public double factorDistance02(double lat, double lon) {
-        return fieldData.factorDistance(docId, DistanceUnit.MILES, lat, lon) + 1;
-    }
-
-    public double factorDistance13(double lat, double lon) {
-        return fieldData.factorDistance(docId, DistanceUnit.MILES, lat, lon) + 2;
-    }
-
-    public double arcDistance(double lat, double lon) {
-        return fieldData.arcDistance(docId, DistanceUnit.MILES, lat, lon);
-    }
-
-    public double arcDistanceWithDefault(double lat, double lon, double defaultValue) {
-        if (!fieldData.hasValue(docId)) {
-            return defaultValue;
-        }
-        return fieldData.arcDistance(docId, DistanceUnit.MILES, lat, lon);
-    }
-
-    public double arcDistanceInKm(double lat, double lon) {
-        return fieldData.arcDistance(docId, DistanceUnit.KILOMETERS, lat, lon);
-    }
-
-    public double arcDistanceInKmWithDefault(double lat, double lon, double defaultValue) {
-        if (!fieldData.hasValue(docId)) {
-            return defaultValue;
-        }
-        return fieldData.arcDistance(docId, DistanceUnit.KILOMETERS, lat, lon);
-    }
-
-    public double distance(double lat, double lon) {
-        return fieldData.distance(docId, DistanceUnit.MILES, lat, lon);
-    }
-
-    public double distanceWithDefault(double lat, double lon, double defaultValue) {
-        if (!fieldData.hasValue(docId)) {
-            return defaultValue;
-        }
-        return fieldData.distance(docId, DistanceUnit.MILES, lat, lon);
-    }
-
-    public double distanceInKm(double lat, double lon) {
-        return fieldData.distance(docId, DistanceUnit.KILOMETERS, lat, lon);
-    }
-
-    public double distanceInKmWithDefault(double lat, double lon, double defaultValue) {
-        if (!fieldData.hasValue(docId)) {
-            return defaultValue;
-        }
-        return fieldData.distance(docId, DistanceUnit.KILOMETERS, lat, lon);
-    }
-
-    public double geohashDistance(String geohash) {
-        return fieldData.distanceGeohash(docId, DistanceUnit.MILES, geohash);
-    }
-
-    public double geohashDistanceWithDefault(String geohash, double defaultValue) {
-        if (!fieldData.hasValue(docId)) {
-            return defaultValue;
-        }
-        return fieldData.distanceGeohash(docId, DistanceUnit.MILES, geohash);
-    }
-
-    public double geohashDistanceInKm(String geohash) {
-        return fieldData.distanceGeohash(docId, DistanceUnit.KILOMETERS, geohash);
-    }
-
-    public double getLat() {
-        return fieldData.latValue(docId);
-    }
-
-    public double getLon() {
-        return fieldData.lonValue(docId);
-    }
-
-    public double[] getLats() {
-        return fieldData.latValues(docId);
-    }
-
-    public double[] getLons() {
-        return fieldData.lonValues(docId);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldData.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldData.java
deleted file mode 100644
index 5e5a1bcb8b7..00000000000
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldData.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper.geo;
-
-import gnu.trove.list.array.TDoubleArrayList;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.support.FieldDataLoader;
-import org.elasticsearch.index.search.geo.GeoDistance;
-import org.elasticsearch.index.search.geo.GeoHashUtils;
-
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class GeoPointFieldData extends FieldData<GeoPointDocFieldData> {
-
-    static ThreadLocal<ThreadLocals.CleanableValue<GeoPoint>> valuesCache = new ThreadLocal<ThreadLocals.CleanableValue<GeoPoint>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<GeoPoint> initialValue() {
-            return new ThreadLocals.CleanableValue<GeoPoint>(new GeoPoint());
-        }
-    };
-
-    static class GeoPointHash {
-        public double lat;
-        public double lon;
-        public String geoHash = "";
-    }
-
-    static ThreadLocal<ThreadLocals.CleanableValue<GeoPointHash>> geoHashCache = new ThreadLocal<ThreadLocals.CleanableValue<GeoPointHash>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<GeoPointHash> initialValue() {
-            return new ThreadLocals.CleanableValue<GeoPointHash>(new GeoPointHash());
-        }
-    };
-
-    public static final GeoPoint[] EMPTY_ARRAY = new GeoPoint[0];
-
-    protected final double[] lat;
-    protected final double[] lon;
-
-    protected GeoPointFieldData(String fieldName, double[] lat, double[] lon) {
-        super(fieldName);
-        this.lat = lat;
-        this.lon = lon;
-    }
-
-    abstract public GeoPoint value(int docId);
-
-    abstract public GeoPoint[] values(int docId);
-
-    abstract public double latValue(int docId);
-
-    abstract public double lonValue(int docId);
-
-    abstract public double[] latValues(int docId);
-
-    abstract public double[] lonValues(int docId);
-
-    public double distance(int docId, DistanceUnit unit, double lat, double lon) {
-        return GeoDistance.PLANE.calculate(latValue(docId), lonValue(docId), lat, lon, unit);
-    }
-
-    public double arcDistance(int docId, DistanceUnit unit, double lat, double lon) {
-        return GeoDistance.ARC.calculate(latValue(docId), lonValue(docId), lat, lon, unit);
-    }
-
-    public double factorDistance(int docId, DistanceUnit unit, double lat, double lon) {
-        return GeoDistance.FACTOR.calculate(latValue(docId), lonValue(docId), lat, lon, unit);
-    }
-
-    public double distanceGeohash(int docId, DistanceUnit unit, String geoHash) {
-        GeoPointHash geoPointHash = geoHashCache.get().get();
-        if (geoPointHash.geoHash != geoHash) {
-            geoPointHash.geoHash = geoHash;
-            double[] decode = GeoHashUtils.decode(geoHash);
-            geoPointHash.lat = decode[0];
-            geoPointHash.lon = decode[1];
-        }
-        return GeoDistance.PLANE.calculate(latValue(docId), lonValue(docId), geoPointHash.lat, geoPointHash.lon, unit);
-    }
-
-    @Override
-    public GeoPointDocFieldData docFieldData(int docId) {
-        return super.docFieldData(docId);
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return (RamUsage.NUM_BYTES_DOUBLE * lat.length + RamUsage.NUM_BYTES_ARRAY_HEADER) +
-                (RamUsage.NUM_BYTES_DOUBLE * lon.length + RamUsage.NUM_BYTES_ARRAY_HEADER);
-    }
-
-    @Override
-    public BytesRef stringValue(int docId) {
-        return new BytesRef(value(docId).geohash());
-    }
-
-    @Override
-    protected GeoPointDocFieldData createFieldData() {
-        return new GeoPointDocFieldData(this);
-    }
-
-    @Override
-    public FieldDataType type() {
-        return GeoPointFieldDataType.TYPE;
-    }
-
-    @Override
-    public void forEachValue(StringValueProc proc) {
-        for (int i = 1; i < lat.length; i++) {
-            proc.onValue(new BytesRef(GeoHashUtils.encode(lat[i], lon[i])));
-        }
-    }
-
-    public void forEachValue(PointValueProc proc) {
-        for (int i = 1; i < lat.length; i++) {
-            GeoPoint point = valuesCache.get().get();
-            point.latlon(lat[i], lon[i]);
-            proc.onValue(point);
-        }
-    }
-
-    public static interface PointValueProc {
-        void onValue(GeoPoint value);
-    }
-
-    public void forEachValue(ValueProc proc) {
-        for (int i = 1; i < lat.length; i++) {
-            proc.onValue(lat[i], lon[i]);
-        }
-    }
-
-    public static interface ValueProc {
-        void onValue(double lat, double lon);
-    }
-
-    public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
-
-    public static interface ValueInDocProc {
-        void onValue(int docId, double lat, double lon);
-    }
-
-    public static GeoPointFieldData load(AtomicReader reader, String field) throws IOException {
-        return FieldDataLoader.load(reader, field, new StringTypeLoader());
-    }
-
-    static class StringTypeLoader extends FieldDataLoader.FreqsTypeLoader<GeoPointFieldData> {
-
-        private final TDoubleArrayList lat = new TDoubleArrayList();
-        private final TDoubleArrayList lon = new TDoubleArrayList();
-
-        StringTypeLoader() {
-            super();
-            // the first one indicates null value
-            lat.add(0);
-            lon.add(0);
-        }
-
-        @Override
-        public void collectTerm(BytesRef term) {
-            // LUCENE 4 UPGRADE: Not nice. We can't operate on a bytesref...
-            String location = term.utf8ToString();
-            int comma = location.indexOf(',');
-            lat.add(Double.parseDouble(location.substring(0, comma)));
-            lon.add(Double.parseDouble(location.substring(comma + 1)));
-
-        }
-
-        @Override
-        public GeoPointFieldData buildSingleValue(String field, int[] ordinals) {
-            return new SingleValueGeoPointFieldData(field, ordinals, lat.toArray(), lon.toArray());
-        }
-
-        @Override
-        public GeoPointFieldData buildMultiValue(String field, int[][] ordinals) {
-            return new MultiValueGeoPointFieldData(field, ordinals, lat.toArray(), lon.toArray());
-        }
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldDataType.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldDataType.java
deleted file mode 100644
index adc1bbb9cb9..00000000000
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldDataType.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper.geo;
-
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.field.data.strings.StringOrdValFieldDataComparator;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class GeoPointFieldDataType implements FieldDataType<GeoPointFieldData> {
-
-    public static final GeoPointFieldDataType TYPE = new GeoPointFieldDataType();
-
-    @Override
-    public ExtendedFieldComparatorSource newFieldComparatorSource(final FieldDataCache cache, final String missing) {
-        return new ExtendedFieldComparatorSource() {
-            @Override
-            public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-                return new StringOrdValFieldDataComparator(numHits, fieldname, sortPos, reversed, cache);
-            }
-
-            @Override
-            public SortField.Type reducedType() {
-                return SortField.Type.STRING;
-            }
-        };
-    }
-
-    @Override
-    public GeoPointFieldData load(AtomicReader reader, String fieldName) throws IOException {
-        return GeoPointFieldData.load(reader, fieldName);
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
index d3caf0cf509..6434366f606 100644
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 import org.elasticsearch.index.mapper.core.DoubleFieldMapper;
@@ -585,15 +584,10 @@ public class GeoPointFieldMapper implements Mapper, ArrayValueMapperParser {
         }
 
         @Override
-        public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+        public org.elasticsearch.index.fielddata.FieldDataType fieldDataType() {
             return new org.elasticsearch.index.fielddata.FieldDataType("geo_point");
         }
 
-        @Override
-        public FieldDataType fieldDataType() {
-            return GeoPointFieldDataType.TYPE;
-        }
-
         public GeoPointFieldMapper geoMapper() {
             return geoMapper;
         }
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
index 2d0b4e258bc..408742bd08a 100644
--- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
@@ -14,6 +14,7 @@ import org.elasticsearch.common.lucene.spatial.prefix.tree.QuadPrefixTree;
 import org.elasticsearch.common.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -148,7 +149,7 @@ public class GeoShapeFieldMapper extends AbstractFieldMapper<String> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         throw new ElasticSearchIllegalArgumentException("field data on geo_shape field is not supported");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java b/src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java
deleted file mode 100644
index a11f222e935..00000000000
--- a/src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper.geo;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.elasticsearch.index.search.geo.GeoHashUtils;
-
-/**
- *
- */
-public class MultiValueGeoPointFieldData extends GeoPointFieldData {
-
-    private static final int VALUE_CACHE_SIZE = 100;
-
-    private static ThreadLocal<ThreadLocals.CleanableValue<GeoPoint[][]>> valuesArrayCache = new ThreadLocal<ThreadLocals.CleanableValue<GeoPoint[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<GeoPoint[][]> initialValue() {
-            GeoPoint[][] value = new GeoPoint[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new GeoPoint[i];
-                for (int j = 0; j < value[i].length; j++) {
-                    value[i][j] = new GeoPoint();
-                }
-            }
-            return new ThreadLocals.CleanableValue<GeoPoint[][]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> valuesLatCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[][]>> valuesLonCache = new ThreadLocal<ThreadLocals.CleanableValue<double[][]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[][]> initialValue() {
-            double[][] value = new double[VALUE_CACHE_SIZE][];
-            for (int i = 0; i < value.length; i++) {
-                value[i] = new double[i];
-            }
-            return new ThreadLocals.CleanableValue<double[][]>(value);
-        }
-    };
-
-    // order with value 0 indicates no value
-    private final int[][] ordinals;
-
-    public MultiValueGeoPointFieldData(String fieldName, int[][] ordinals, double[] lat, double[] lon) {
-        super(fieldName, lat, lon);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        long size = super.computeSizeInBytes();
-        size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
-        for (int[] ordinal : ordinals) {
-            size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-        }
-        return size;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return true;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onMissing(docId);
-                }
-                break;
-            }
-            //LUCENE 4 UPGRADE: Make GeoHashUtils works with BytesRef instead of String
-            proc.onValue(docId, new BytesRef(GeoHashUtils.encode(lat[loc], lon[loc])));
-        }
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc == 0) {
-                break;
-            }
-            proc.onValue(docId, lat[loc], lon[loc]);
-        }
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        for (int i = 0; i < ordinals.length; i++) {
-            int loc = ordinals[i][docId];
-            if (loc == 0) {
-                if (i == 0) {
-                    proc.onOrdinal(docId, 0);
-                }
-                break;
-            }
-            proc.onOrdinal(docId, loc);
-        }
-    }
-
-    @Override
-    public GeoPoint value(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                GeoPoint point = valuesCache.get().get();
-                point.latlon(lat[loc], lon[loc]);
-                return point;
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public GeoPoint[] values(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return EMPTY_ARRAY;
-        }
-        GeoPoint[] points;
-        if (length < VALUE_CACHE_SIZE) {
-            points = valuesArrayCache.get().get()[length];
-            for (int i = 0; i < length; i++) {
-                int loc = ordinals[i][docId];
-                points[i].latlon(lat[loc], lon[loc]);
-            }
-        } else {
-            points = new GeoPoint[length];
-            for (int i = 0; i < length; i++) {
-                int loc = ordinals[i][docId];
-                points[i] = new GeoPoint(lat[loc], lon[loc]);
-            }
-        }
-        return points;
-    }
-
-    @Override
-    public double latValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return lat[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public double lonValue(int docId) {
-        for (int[] ordinal : ordinals) {
-            int loc = ordinal[docId];
-            if (loc != 0) {
-                return lon[loc];
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public double[] latValues(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] == 0) {
-                break;
-            }
-            length++;
-        }
-        if (length == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = valuesLatCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        for (int i = 0; i < length; i++) {
-            doubles[i] = lat[ordinals[i][docId]];
-        }
-        return doubles;
-    }
-
-    @Override
-    public double[] lonValues(int docId) {
-        int length = 0;
-        for (int[] ordinal : ordinals) {
-            if (ordinal[docId] != 0) {
-                length++;
-            }
-        }
-        if (length == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] doubles;
-        if (length < VALUE_CACHE_SIZE) {
-            doubles = valuesLonCache.get().get()[length];
-        } else {
-            doubles = new double[length];
-        }
-        for (int i = 0; i < length; i++) {
-            doubles[i] = lon[ordinals[i][docId]];
-        }
-        return doubles;
-    }
-}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/SingleValueGeoPointFieldData.java b/src/main/java/org/elasticsearch/index/mapper/geo/SingleValueGeoPointFieldData.java
deleted file mode 100644
index fb7f78e352a..00000000000
--- a/src/main/java/org/elasticsearch/index/mapper/geo/SingleValueGeoPointFieldData.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper.geo;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.RamUsage;
-import org.elasticsearch.common.util.concurrent.ThreadLocals;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.elasticsearch.index.search.geo.GeoHashUtils;
-
-/**
- *
- */
-public class SingleValueGeoPointFieldData extends GeoPointFieldData {
-
-    private static ThreadLocal<ThreadLocals.CleanableValue<GeoPoint[]>> valuesArrayCache = new ThreadLocal<ThreadLocals.CleanableValue<GeoPoint[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<GeoPoint[]> initialValue() {
-            GeoPoint[] value = new GeoPoint[1];
-            value[0] = new GeoPoint();
-            return new ThreadLocals.CleanableValue<GeoPoint[]>(value);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> valuesLatCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-    private ThreadLocal<ThreadLocals.CleanableValue<double[]>> valuesLonCache = new ThreadLocal<ThreadLocals.CleanableValue<double[]>>() {
-        @Override
-        protected ThreadLocals.CleanableValue<double[]> initialValue() {
-            return new ThreadLocals.CleanableValue<double[]>(new double[1]);
-        }
-    };
-
-
-    // order with value 0 indicates no value
-    private final int[] ordinals;
-
-    public SingleValueGeoPointFieldData(String fieldName, int[] ordinals, double[] lat, double[] lon) {
-        super(fieldName, lat, lon);
-        this.ordinals = ordinals;
-    }
-
-    @Override
-    protected long computeSizeInBytes() {
-        return super.computeSizeInBytes() +
-                RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
-    }
-
-    @Override
-    public boolean multiValued() {
-        return false;
-    }
-
-    @Override
-    public boolean hasValue(int docId) {
-        return ordinals[docId] != 0;
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, StringValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            proc.onMissing(docId);
-            return;
-        }
-        //LUCENE 4 UPGRADE: Make GeoHashUtils works with BytesRef instead of String
-        proc.onValue(docId, new BytesRef(GeoHashUtils.encode(lat[loc], lon[loc])));
-    }
-
-    @Override
-    public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
-        proc.onOrdinal(docId, ordinals[docId]);
-    }
-
-    @Override
-    public void forEachValueInDoc(int docId, ValueInDocProc proc) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return;
-        }
-        proc.onValue(docId, lat[loc], lon[loc]);
-    }
-
-    @Override
-    public GeoPoint value(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return null;
-        }
-        GeoPoint point = valuesCache.get().get();
-        point.latlon(lat[loc], lon[loc]);
-        return point;
-    }
-
-    @Override
-    public GeoPoint[] values(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return EMPTY_ARRAY;
-        }
-        GeoPoint[] ret = valuesArrayCache.get().get();
-        ret[0].latlon(lat[loc], lon[loc]);
-        return ret;
-    }
-
-    @Override
-    public double latValue(int docId) {
-        return lat[ordinals[docId]];
-    }
-
-    @Override
-    public double lonValue(int docId) {
-        return lon[ordinals[docId]];
-    }
-
-    @Override
-    public double[] latValues(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = valuesLatCache.get().get();
-        ret[0] = lat[loc];
-        return ret;
-    }
-
-    @Override
-    public double[] lonValues(int docId) {
-        int loc = ordinals[docId];
-        if (loc == 0) {
-            return DoubleFieldData.EMPTY_DOUBLE_ARRAY;
-        }
-        double[] ret = valuesLonCache.get().get();
-        ret[0] = lon[loc];
-        return ret;
-    }
-}
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
index d84d695de7c..32b573c238d 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
@@ -153,7 +153,7 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements Interna
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType() {
         throw new ElasticSearchIllegalArgumentException("using field data on _all field is not supported");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
index 31ce24a80a1..54c7ebe0c36 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/BoostFieldMapper.java
@@ -36,7 +36,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericFloatAnalyzer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -134,7 +134,7 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         throw new ElasticSearchIllegalArgumentException("not implemented");
     }
 
@@ -280,11 +280,6 @@ public class BoostFieldMapper extends NumberFieldMapper<Float> implements Intern
         return value;
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.FLOAT;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
index fe0cf421028..d358abd9269 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
@@ -144,7 +144,7 @@ public class IdFieldMapper extends AbstractFieldMapper<String> implements Intern
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         return new FieldDataType("string");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
index d278fab1706..8f63659fca4 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
@@ -128,7 +128,7 @@ public class IndexFieldMapper extends AbstractFieldMapper<String> implements Int
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         return new FieldDataType("string");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
index 9eb1f6d96de..721bcf54ee4 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
@@ -136,7 +136,7 @@ public class ParentFieldMapper extends AbstractFieldMapper<Uid> implements Inter
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         return new FieldDataType("string");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
index c8b81540a6e..e424f82174a 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
@@ -131,7 +131,7 @@ public class RoutingFieldMapper extends AbstractFieldMapper<String> implements I
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         return new FieldDataType("string");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
index 704f9cb1797..c06d8cdbfb8 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
@@ -43,6 +43,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
 
@@ -220,7 +221,7 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         throw new ElasticSearchIllegalArgumentException("field data on _source field is not supported");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index 77283d8a87d..09d8187de8f 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -111,7 +111,7 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements Inte
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         return new FieldDataType("string");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index 2c6341157d1..9be22de4eee 100644
--- a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -128,7 +128,7 @@ public class UidFieldMapper extends AbstractFieldMapper<Uid> implements Internal
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public FieldDataType fieldDataType() {
         return new FieldDataType("string", "paged_bytes");
     }
 
diff --git a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index f2d07688c7f..4d43435c45b 100644
--- a/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -36,7 +36,6 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericAnalyzer;
 import org.elasticsearch.index.analysis.NumericTokenizer;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
-import org.elasticsearch.index.field.data.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.*;
@@ -155,7 +154,7 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
     }
 
     @Override
-    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType2() {
+    public org.elasticsearch.index.fielddata.FieldDataType fieldDataType() {
         return new org.elasticsearch.index.fielddata.FieldDataType("long");
     }
 
@@ -296,11 +295,6 @@ public class IpFieldMapper extends NumberFieldMapper<Long> {
         return new LongFieldMapper.CustomLongNumericField(this, value, fieldType);
     }
 
-    @Override
-    public FieldDataType fieldDataType() {
-        return FieldDataType.DefaultTypes.LONG;
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
diff --git a/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceDataComparator.java b/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceDataComparator.java
deleted file mode 100644
index 639e608f2fd..00000000000
--- a/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceDataComparator.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.search.geo;
-
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.SortField;
-import org.elasticsearch.ElasticSearchIllegalArgumentException;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.cache.field.data.FieldDataCache;
-import org.elasticsearch.index.field.data.FieldDataType;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldData;
-import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType;
-
-import java.io.IOException;
-
-/**
- *
- */
-// LUCENE MONITOR: Monitor against FieldComparator.Double
-public class GeoDistanceDataComparator extends FieldComparator<Double> {
-
-    public static FieldDataType.ExtendedFieldComparatorSource comparatorSource(String fieldName, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
-                                                                               FieldDataCache fieldDataCache, MapperService mapperService) {
-        return new InnerSource(fieldName, lat, lon, unit, geoDistance, fieldDataCache, mapperService);
-    }
-
-    static class InnerSource extends FieldDataType.ExtendedFieldComparatorSource {
-
-        protected final String fieldName;
-
-        protected final double lat;
-
-        protected final double lon;
-
-        protected final DistanceUnit unit;
-
-        protected final GeoDistance geoDistance;
-
-        protected final FieldDataCache fieldDataCache;
-
-        private final MapperService mapperService;
-
-        private InnerSource(String fieldName, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
-                            FieldDataCache fieldDataCache, MapperService mapperService) {
-            this.fieldName = fieldName;
-            this.lat = lat;
-            this.lon = lon;
-            this.unit = unit;
-            this.geoDistance = geoDistance;
-            this.fieldDataCache = fieldDataCache;
-            this.mapperService = mapperService;
-        }
-
-        @Override
-        public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
-            return new GeoDistanceDataComparator(numHits, fieldname, lat, lon, unit, geoDistance, fieldDataCache, mapperService);
-        }
-
-        @Override
-        public SortField.Type reducedType() {
-            return SortField.Type.DOUBLE;
-        }
-    }
-
-    protected final String fieldName;
-
-    protected final String indexFieldName;
-
-    protected final double lat;
-
-    protected final double lon;
-
-    protected final DistanceUnit unit;
-
-    protected final GeoDistance geoDistance;
-    protected final GeoDistance.FixedSourceDistance fixedSourceDistance;
-
-    protected final FieldDataCache fieldDataCache;
-
-    protected GeoPointFieldData fieldData;
-
-
-    private final double[] values;
-    private double bottom;
-
-    public GeoDistanceDataComparator(int numHits, String fieldName, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance,
-                                     FieldDataCache fieldDataCache, MapperService mapperService) {
-        values = new double[numHits];
-
-        this.fieldName = fieldName;
-        this.lat = lat;
-        this.lon = lon;
-        this.unit = unit;
-        this.geoDistance = geoDistance;
-        this.fieldDataCache = fieldDataCache;
-
-        this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, unit);
-
-        FieldMapper mapper = mapperService.smartNameFieldMapper(fieldName);
-        if (mapper == null) {
-            throw new ElasticSearchIllegalArgumentException("No mapping found for field [" + fieldName + "] for geo distance sort");
-        }
-        if (mapper.fieldDataType() != GeoPointFieldDataType.TYPE) {
-            throw new ElasticSearchIllegalArgumentException("field [" + fieldName + "] is not a geo_point field");
-        }
-        this.indexFieldName = mapper.names().indexName();
-    }
-
-    @Override
-    public GeoDistanceDataComparator setNextReader(AtomicReaderContext context) throws IOException {
-        fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, context.reader(), indexFieldName);
-        return this;
-    }
-
-    @Override
-    public int compare(int slot1, int slot2) {
-        final double v1 = values[slot1];
-        final double v2 = values[slot2];
-        if (v1 > v2) {
-            return 1;
-        } else if (v1 < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareBottom(int doc) {
-        double distance;
-        if (!fieldData.hasValue(doc)) {
-            // is this true? push this to the "end"
-            distance = Double.MAX_VALUE;
-        } else {
-            distance = fixedSourceDistance.calculate(fieldData.latValue(doc), fieldData.lonValue(doc));
-        }
-        final double v2 = distance;
-        if (bottom > v2) {
-            return 1;
-        } else if (bottom < v2) {
-            return -1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public int compareDocToValue(int doc, Double distance2) throws IOException {
-        double distance1;
-        if (!fieldData.hasValue(doc)) {
-            // is this true? push this to the "end"
-            distance1 = Double.MAX_VALUE;
-        } else {
-            distance1 = fixedSourceDistance.calculate(fieldData.latValue(doc), fieldData.lonValue(doc));
-        }
-        return (int) (distance1 - distance2);
-    }
-
-    @Override
-    public void copy(int slot, int doc) {
-        double distance;
-        if (!fieldData.hasValue(doc)) {
-            // is this true? push this to the "end"
-            distance = Double.MAX_VALUE;
-        } else {
-            distance = fixedSourceDistance.calculate(fieldData.latValue(doc), fieldData.lonValue(doc));
-        }
-        values[slot] = distance;
-    }
-
-    @Override
-    public void setBottom(final int bottom) {
-        this.bottom = values[bottom];
-    }
-
-    @Override
-    public Double value(int slot) {
-        return values[slot];
-    }
-}
diff --git a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
index 7eddf51ca89..41dcb330c4c 100644
--- a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
+++ b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
@@ -24,8 +24,6 @@ import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.index.field.data.DocFieldData;
-import org.elasticsearch.index.field.data.NumericDocFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -83,14 +81,6 @@ public class DocLookup implements Map {
         this.docId = docId;
     }
 
-    public <T extends DocFieldData> T field(String key) {
-        return (T) get(key);
-    }
-
-    public <T extends NumericDocFieldData> T numeric(String key) {
-        return (T) get(key);
-    }
-
     public float score() throws IOException {
         return scorer.score();
     }
diff --git a/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java b/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java
index b909a0fdd39..1aea4853e0c 100644
--- a/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java
+++ b/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java
@@ -22,8 +22,8 @@ package org.elasticsearch.search.sort;
 import org.apache.lucene.search.FieldComparatorSource;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.field.function.sort.DoubleFieldsFunctionDataComparator;
-import org.elasticsearch.index.field.function.sort.StringFieldsFunctionDataComparator;
+import org.elasticsearch.index.fielddata.fieldcomparator.DoubleScriptDataComparator;
+import org.elasticsearch.index.fielddata.fieldcomparator.StringScriptDataComparator;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.internal.SearchContext;
@@ -81,9 +81,9 @@ public class ScriptSortParser implements SortParser {
         SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, params);
         FieldComparatorSource fieldComparatorSource;
         if ("string".equals(type)) {
-            fieldComparatorSource = StringFieldsFunctionDataComparator.comparatorSource(searchScript);
+            fieldComparatorSource = StringScriptDataComparator.comparatorSource(searchScript);
         } else if ("number".equals(type)) {
-            fieldComparatorSource = DoubleFieldsFunctionDataComparator.comparatorSource(searchScript);
+            fieldComparatorSource = DoubleScriptDataComparator.comparatorSource(searchScript);
         } else {
             throw new SearchParseException(context, "custom script sort type [" + type + "] not supported");
         }
diff --git a/src/test/java/org/elasticsearch/test/unit/index/field/data/doubles/DoubleFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/field/data/doubles/DoubleFieldDataTests.java
deleted file mode 100644
index bd8c8e45c40..00000000000
--- a/src/test/java/org/elasticsearch/test/unit/index/field/data/doubles/DoubleFieldDataTests.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.unit.index.field.data.doubles;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.*;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.field.data.doubles.DoubleFieldData;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-public class DoubleFieldDataTests {
-
-    @Test
-    public void intFieldDataTests() throws Exception {
-        Directory dir = new RAMDirectory();
-        IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
-
-        Document document = new Document();
-        document.add(new DoubleField("svalue", 4, Field.Store.NO));
-        document.add(new DoubleField("mvalue", 104, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new DoubleField("svalue", 3, Field.Store.NO));
-        document.add(new DoubleField("mvalue", 104, Field.Store.NO));
-        document.add(new DoubleField("mvalue", 105, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new DoubleField("svalue", 7, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new DoubleField("mvalue", 102, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new DoubleField("svalue", 4, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(indexWriter, true));
-
-        DoubleFieldData sFieldData = DoubleFieldData.load(reader, "svalue");
-        DoubleFieldData mFieldData = DoubleFieldData.load(reader, "mvalue");
-
-        assertThat(sFieldData.fieldName(), equalTo("svalue"));
-        assertThat(sFieldData.multiValued(), equalTo(false));
-
-        assertThat(mFieldData.fieldName(), equalTo("mvalue"));
-        assertThat(mFieldData.multiValued(), equalTo(true));
-
-        // svalue
-        assertThat(sFieldData.hasValue(0), equalTo(true));
-        assertThat(sFieldData.value(0), equalTo(4d));
-        assertThat(sFieldData.docFieldData(0).getValue(), equalTo(4d));
-        assertThat(sFieldData.values(0).length, equalTo(1));
-        assertThat(sFieldData.values(0)[0], equalTo(4d));
-        assertThat(sFieldData.docFieldData(0).getValues()[0], equalTo(4d));
-
-        assertThat(sFieldData.hasValue(1), equalTo(true));
-        assertThat(sFieldData.value(1), equalTo(3d));
-        assertThat(sFieldData.docFieldData(1).getValue(), equalTo(3d));
-        assertThat(sFieldData.values(1).length, equalTo(1));
-        assertThat(sFieldData.values(1)[0], equalTo(3d));
-        assertThat(sFieldData.docFieldData(1).getValues()[0], equalTo(3d));
-
-        assertThat(sFieldData.hasValue(2), equalTo(true));
-        assertThat(sFieldData.value(2), equalTo(7d));
-        assertThat(sFieldData.docFieldData(2).getValue(), equalTo(7d));
-        assertThat(sFieldData.values(2).length, equalTo(1));
-        assertThat(sFieldData.values(2)[0], equalTo(7d));
-        assertThat(sFieldData.docFieldData(2).getValues()[0], equalTo(7d));
-
-        assertThat(sFieldData.hasValue(3), equalTo(false));
-        assertThat(sFieldData.docFieldData(3).isEmpty(), equalTo(true));
-
-        assertThat(sFieldData.hasValue(4), equalTo(true));
-        assertThat(sFieldData.value(4), equalTo(4d));
-        assertThat(sFieldData.values(4).length, equalTo(1));
-        assertThat(sFieldData.values(4)[0], equalTo(4d));
-
-        // check order is correct
-        final ArrayList<Double> values = new ArrayList<Double>();
-        sFieldData.forEachValue(new DoubleFieldData.ValueProc() {
-            @Override
-            public void onValue(double value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo(3d));
-        assertThat(values.get(1), equalTo(4d));
-        assertThat(values.get(2), equalTo(7d));
-
-        // mvalue
-        assertThat(mFieldData.hasValue(0), equalTo(true));
-        assertThat(mFieldData.docFieldData(0).isEmpty(), equalTo(false));
-        assertThat(mFieldData.value(0), equalTo(104d));
-        assertThat(mFieldData.docFieldData(0).getValue(), equalTo(104d));
-        assertThat(mFieldData.values(0).length, equalTo(1));
-        assertThat(mFieldData.values(0)[0], equalTo(104d));
-        assertThat(mFieldData.docFieldData(0).getValues()[0], equalTo(104d));
-
-        assertThat(mFieldData.hasValue(1), equalTo(true));
-        assertThat(mFieldData.value(1), equalTo(104d));
-        assertThat(mFieldData.values(1).length, equalTo(2));
-        assertThat(mFieldData.values(1)[0], equalTo(104d));
-        assertThat(mFieldData.values(1)[1], equalTo(105d));
-
-        assertThat(mFieldData.hasValue(2), equalTo(false));
-
-        assertThat(mFieldData.hasValue(3), equalTo(true));
-        assertThat(mFieldData.value(3), equalTo(102d));
-        assertThat(mFieldData.values(3).length, equalTo(1));
-        assertThat(mFieldData.values(3)[0], equalTo(102d));
-
-        assertThat(mFieldData.hasValue(4), equalTo(false));
-
-        indexWriter.close();
-
-        // check order is correct
-        values.clear();
-        mFieldData.forEachValue(new DoubleFieldData.ValueProc() {
-            @Override
-            public void onValue(double value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo(102d));
-        assertThat(values.get(1), equalTo(104d));
-        assertThat(values.get(2), equalTo(105d));
-    }
-}
\ No newline at end of file
diff --git a/src/test/java/org/elasticsearch/test/unit/index/field/data/floats/FloatFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/field/data/floats/FloatFieldDataTests.java
deleted file mode 100644
index 08ca48d6ad9..00000000000
--- a/src/test/java/org/elasticsearch/test/unit/index/field/data/floats/FloatFieldDataTests.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.unit.index.field.data.floats;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.field.data.floats.FloatFieldData;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-public class FloatFieldDataTests {
-
-    @Test
-    public void intFieldDataTests() throws Exception {
-        Directory dir = new RAMDirectory();
-        IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
-
-        Document document = new Document();
-        document.add(new FloatField("svalue", 4, Field.Store.NO));
-        document.add(new FloatField("mvalue", 104, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new FloatField("svalue", 3, Field.Store.NO));
-        document.add(new FloatField("mvalue", 104, Field.Store.NO));
-        document.add(new FloatField("mvalue", 105, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new FloatField("svalue", 7, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new FloatField("mvalue", 102, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new FloatField("svalue", 4, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(indexWriter, true));
-
-        FloatFieldData sFieldData = FloatFieldData.load(reader, "svalue");
-        FloatFieldData mFieldData = FloatFieldData.load(reader, "mvalue");
-
-        assertThat(sFieldData.fieldName(), equalTo("svalue"));
-        assertThat(sFieldData.multiValued(), equalTo(false));
-
-        assertThat(mFieldData.fieldName(), equalTo("mvalue"));
-        assertThat(mFieldData.multiValued(), equalTo(true));
-
-        // svalue
-        assertThat(sFieldData.hasValue(0), equalTo(true));
-        assertThat(sFieldData.docFieldData(0).isEmpty(), equalTo(false));
-        assertThat(sFieldData.value(0), equalTo(4f));
-        assertThat(sFieldData.docFieldData(0).getValue(), equalTo(4f));
-        assertThat(sFieldData.values(0).length, equalTo(1));
-        assertThat(sFieldData.docFieldData(0).getValues().length, equalTo(1));
-        assertThat(sFieldData.docFieldData(0).getValues()[0], equalTo(4f));
-
-        assertThat(sFieldData.hasValue(1), equalTo(true));
-        assertThat(sFieldData.value(1), equalTo(3f));
-        assertThat(sFieldData.values(1).length, equalTo(1));
-        assertThat(sFieldData.values(1)[0], equalTo(3f));
-
-        assertThat(sFieldData.hasValue(2), equalTo(true));
-        assertThat(sFieldData.value(2), equalTo(7f));
-        assertThat(sFieldData.values(2).length, equalTo(1));
-        assertThat(sFieldData.values(2)[0], equalTo(7f));
-
-        assertThat(sFieldData.hasValue(3), equalTo(false));
-
-        assertThat(sFieldData.hasValue(4), equalTo(true));
-        assertThat(sFieldData.value(4), equalTo(4f));
-        assertThat(sFieldData.values(4).length, equalTo(1));
-        assertThat(sFieldData.values(4)[0], equalTo(4f));
-
-        // check order is correct
-        final ArrayList<Float> values = new ArrayList<Float>();
-        sFieldData.forEachValue(new FloatFieldData.ValueProc() {
-            @Override
-            public void onValue(float value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo(3f));
-        assertThat(values.get(1), equalTo(4f));
-        assertThat(values.get(2), equalTo(7f));
-
-
-        // mvalue
-        assertThat(mFieldData.hasValue(0), equalTo(true));
-        assertThat(mFieldData.docFieldData(0).isEmpty(), equalTo(false));
-        assertThat(mFieldData.value(0), equalTo(104f));
-        assertThat(mFieldData.docFieldData(0).getValue(), equalTo(104f));
-        assertThat(mFieldData.values(0).length, equalTo(1));
-        assertThat(mFieldData.docFieldData(0).getValues().length, equalTo(1));
-        assertThat(mFieldData.values(0)[0], equalTo(104f));
-        assertThat(mFieldData.docFieldData(0).getValues()[0], equalTo(104f));
-
-        assertThat(mFieldData.hasValue(1), equalTo(true));
-        assertThat(mFieldData.value(1), equalTo(104f));
-        assertThat(mFieldData.values(1).length, equalTo(2));
-        assertThat(mFieldData.docFieldData(1).getValues().length, equalTo(2));
-        assertThat(mFieldData.values(1)[0], equalTo(104f));
-        assertThat(mFieldData.docFieldData(1).getValues()[0], equalTo(104f));
-        assertThat(mFieldData.docFieldData(1).getValues()[1], equalTo(105f));
-
-        assertThat(mFieldData.hasValue(2), equalTo(false));
-
-        assertThat(mFieldData.hasValue(3), equalTo(true));
-        assertThat(mFieldData.value(3), equalTo(102f));
-        assertThat(mFieldData.values(3).length, equalTo(1));
-        assertThat(mFieldData.values(3)[0], equalTo(102f));
-
-        assertThat(mFieldData.hasValue(4), equalTo(false));
-
-        indexWriter.close();
-
-        // check order is correct
-        values.clear();
-        mFieldData.forEachValue(new FloatFieldData.ValueProc() {
-            @Override
-            public void onValue(float value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo(102f));
-        assertThat(values.get(1), equalTo(104f));
-        assertThat(values.get(2), equalTo(105f));
-    }
-}
\ No newline at end of file
diff --git a/src/test/java/org/elasticsearch/test/unit/index/field/data/ints/IntFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/field/data/ints/IntFieldDataTests.java
deleted file mode 100644
index daa7a67a7ad..00000000000
--- a/src/test/java/org/elasticsearch/test/unit/index/field/data/ints/IntFieldDataTests.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.unit.index.field.data.ints;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.field.data.ints.IntFieldData;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-public class IntFieldDataTests {
-
-    @Test
-    public void intFieldDataTests() throws Exception {
-        Directory dir = new RAMDirectory();
-        IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
-
-        Document document = new Document();
-        document.add(new IntField("svalue", 4, Field.Store.NO));
-        document.add(new IntField("mvalue", 104, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("svalue", 3, Field.Store.NO));
-        document.add(new IntField("mvalue", 104, Field.Store.NO));
-        document.add(new IntField("mvalue", 105, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("svalue", 7, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("mvalue", 102, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("svalue", 4, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(indexWriter, true));
-
-        IntFieldData sFieldData = IntFieldData.load(reader, "svalue");
-        IntFieldData mFieldData = IntFieldData.load(reader, "mvalue");
-
-        assertThat(sFieldData.fieldName(), equalTo("svalue"));
-        assertThat(sFieldData.multiValued(), equalTo(false));
-
-        assertThat(mFieldData.fieldName(), equalTo("mvalue"));
-        assertThat(mFieldData.multiValued(), equalTo(true));
-
-        // svalue
-        assertThat(sFieldData.hasValue(0), equalTo(true));
-        assertThat(sFieldData.docFieldData(0).isEmpty(), equalTo(false));
-        assertThat(sFieldData.value(0), equalTo(4));
-        assertThat(sFieldData.docFieldData(0).getValue(), equalTo(4));
-        assertThat(sFieldData.values(0).length, equalTo(1));
-        assertThat(sFieldData.docFieldData(0).getValues().length, equalTo(1));
-        assertThat(sFieldData.docFieldData(0).getValues()[0], equalTo(4));
-
-        assertThat(sFieldData.hasValue(1), equalTo(true));
-        assertThat(sFieldData.value(1), equalTo(3));
-        assertThat(sFieldData.values(1).length, equalTo(1));
-        assertThat(sFieldData.values(1)[0], equalTo(3));
-
-        assertThat(sFieldData.hasValue(2), equalTo(true));
-        assertThat(sFieldData.value(2), equalTo(7));
-        assertThat(sFieldData.values(2).length, equalTo(1));
-        assertThat(sFieldData.values(2)[0], equalTo(7));
-
-        assertThat(sFieldData.hasValue(3), equalTo(false));
-
-        assertThat(sFieldData.hasValue(4), equalTo(true));
-        assertThat(sFieldData.value(4), equalTo(4));
-        assertThat(sFieldData.values(4).length, equalTo(1));
-        assertThat(sFieldData.values(4)[0], equalTo(4));
-
-        // check order is correct
-        final ArrayList<Integer> values = new ArrayList<Integer>();
-        sFieldData.forEachValue(new IntFieldData.ValueProc() {
-            @Override
-            public void onValue(int value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo(3));
-        assertThat(values.get(1), equalTo(4));
-        assertThat(values.get(2), equalTo(7));
-
-        // mvalue
-        assertThat(mFieldData.hasValue(0), equalTo(true));
-        assertThat(mFieldData.value(0), equalTo(104));
-        assertThat(mFieldData.values(0).length, equalTo(1));
-        assertThat(mFieldData.values(0)[0], equalTo(104));
-
-        assertThat(mFieldData.hasValue(1), equalTo(true));
-        assertThat(mFieldData.value(1), equalTo(104));
-        assertThat(mFieldData.values(1).length, equalTo(2));
-        assertThat(mFieldData.values(1)[0], equalTo(104));
-        assertThat(mFieldData.values(1)[1], equalTo(105));
-
-        assertThat(mFieldData.hasValue(2), equalTo(false));
-
-        assertThat(mFieldData.hasValue(3), equalTo(true));
-        assertThat(mFieldData.value(3), equalTo(102));
-        assertThat(mFieldData.values(3).length, equalTo(1));
-        assertThat(mFieldData.values(3)[0], equalTo(102));
-
-        assertThat(mFieldData.hasValue(4), equalTo(false));
-
-        indexWriter.close();
-
-        // check order is correct
-        values.clear();
-        mFieldData.forEachValue(new IntFieldData.ValueProc() {
-            @Override
-            public void onValue(int value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-        assertThat(values.get(0), equalTo(102));
-        assertThat(values.get(1), equalTo(104));
-        assertThat(values.get(2), equalTo(105));
-    }
-}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/field/data/longs/LongFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/field/data/longs/LongFieldDataTests.java
deleted file mode 100644
index 08fa28f5a80..00000000000
--- a/src/test/java/org/elasticsearch/test/unit/index/field/data/longs/LongFieldDataTests.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.unit.index.field.data.longs;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.LongField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.field.data.longs.LongFieldData;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-public class LongFieldDataTests {
-
-    @Test
-    public void intFieldDataTests() throws Exception {
-        Directory dir = new RAMDirectory();
-        IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
-
-        Document document = new Document();
-        document.add(new LongField("svalue", 4, Field.Store.NO));
-        document.add(new LongField("mvalue", 104, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new LongField("svalue", 3, Field.Store.NO));
-        document.add(new LongField("mvalue", 104, Field.Store.NO));
-        document.add(new LongField("mvalue", 105, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new LongField("svalue", 7, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new LongField("mvalue", 102, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new LongField("svalue", 4, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(indexWriter, true));
-
-        LongFieldData sFieldData = LongFieldData.load(reader, "svalue");
-        LongFieldData mFieldData = LongFieldData.load(reader, "mvalue");
-
-        assertThat(sFieldData.fieldName(), equalTo("svalue"));
-        assertThat(sFieldData.multiValued(), equalTo(false));
-
-        assertThat(mFieldData.fieldName(), equalTo("mvalue"));
-        assertThat(mFieldData.multiValued(), equalTo(true));
-
-        // svalue
-        assertThat(sFieldData.hasValue(0), equalTo(true));
-        assertThat(sFieldData.docFieldData(0).isEmpty(), equalTo(false));
-        assertThat(sFieldData.value(0), equalTo(4l));
-        assertThat(sFieldData.date(0).getMillis(), equalTo(4l));
-        assertThat(sFieldData.docFieldData(0).getValue(), equalTo(4l));
-        assertThat(sFieldData.values(0).length, equalTo(1));
-        assertThat(sFieldData.docFieldData(0).getValues().length, equalTo(1));
-        assertThat(sFieldData.values(0)[0], equalTo(4l));
-        assertThat(sFieldData.dates(0)[0].getMillis(), equalTo(4l));
-        assertThat(sFieldData.docFieldData(0).getValues()[0], equalTo(4l));
-
-        assertThat(sFieldData.hasValue(1), equalTo(true));
-        assertThat(sFieldData.value(1), equalTo(3l));
-        assertThat(sFieldData.values(1).length, equalTo(1));
-        assertThat(sFieldData.values(1)[0], equalTo(3l));
-
-        assertThat(sFieldData.hasValue(2), equalTo(true));
-        assertThat(sFieldData.value(2), equalTo(7l));
-        assertThat(sFieldData.values(2).length, equalTo(1));
-        assertThat(sFieldData.values(2)[0], equalTo(7l));
-
-        assertThat(sFieldData.hasValue(3), equalTo(false));
-
-        assertThat(sFieldData.hasValue(4), equalTo(true));
-        assertThat(sFieldData.value(4), equalTo(4l));
-        assertThat(sFieldData.values(4).length, equalTo(1));
-        assertThat(sFieldData.values(4)[0], equalTo(4l));
-
-        // check order is correct
-        final ArrayList<Long> values = new ArrayList<Long>();
-        sFieldData.forEachValue(new LongFieldData.ValueProc() {
-            @Override
-            public void onValue(long value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo(3l));
-        assertThat(values.get(1), equalTo(4l));
-        assertThat(values.get(2), equalTo(7l));
-
-
-        // mvalue
-        assertThat(mFieldData.hasValue(0), equalTo(true));
-        assertThat(mFieldData.value(0), equalTo(104l));
-        assertThat(mFieldData.values(0).length, equalTo(1));
-        assertThat(mFieldData.values(0)[0], equalTo(104l));
-
-        assertThat(mFieldData.hasValue(1), equalTo(true));
-        assertThat(mFieldData.value(1), equalTo(104l));
-        assertThat(mFieldData.date(1).getMillis(), equalTo(104l));
-        assertThat(mFieldData.values(1).length, equalTo(2));
-        assertThat(mFieldData.values(1)[0], equalTo(104l));
-        assertThat(mFieldData.dates(1)[0].getMillis(), equalTo(104l));
-        assertThat(mFieldData.values(1)[1], equalTo(105l));
-        assertThat(mFieldData.dates(1)[1].getMillis(), equalTo(105l));
-
-        assertThat(mFieldData.hasValue(2), equalTo(false));
-
-        assertThat(mFieldData.hasValue(3), equalTo(true));
-        assertThat(mFieldData.value(3), equalTo(102l));
-        assertThat(mFieldData.values(3).length, equalTo(1));
-        assertThat(mFieldData.values(3)[0], equalTo(102l));
-
-        assertThat(mFieldData.hasValue(4), equalTo(false));
-
-        indexWriter.close();
-
-        // check order is correct
-        values.clear();
-        mFieldData.forEachValue(new LongFieldData.ValueProc() {
-            @Override
-            public void onValue(long value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo(102l));
-        assertThat(values.get(1), equalTo(104l));
-        assertThat(values.get(2), equalTo(105l));
-    }
-}
\ No newline at end of file
diff --git a/src/test/java/org/elasticsearch/test/unit/index/field/data/shorts/ShortFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/field/data/shorts/ShortFieldDataTests.java
deleted file mode 100644
index 27b34e5d868..00000000000
--- a/src/test/java/org/elasticsearch/test/unit/index/field/data/shorts/ShortFieldDataTests.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.unit.index.field.data.shorts;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.field.data.shorts.ShortFieldData;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-public class ShortFieldDataTests {
-
-    @Test
-    public void intFieldDataTests() throws Exception {
-        Directory dir = new RAMDirectory();
-        IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
-
-        Document document = new Document();
-        document.add(new IntField("svalue", 4, Field.Store.NO));
-        document.add(new IntField("mvalue", 104, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("svalue", 3, Field.Store.NO));
-        document.add(new IntField("mvalue", 104, Field.Store.NO));
-        document.add(new IntField("mvalue", 105, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("svalue", 7, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("mvalue", 102, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new IntField("svalue", 4, Field.Store.NO));
-        indexWriter.addDocument(document);
-
-        AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(indexWriter, true));
-
-        ShortFieldData sFieldData = ShortFieldData.load(reader, "svalue");
-        ShortFieldData mFieldData = ShortFieldData.load(reader, "mvalue");
-
-        assertThat(sFieldData.fieldName(), equalTo("svalue"));
-        assertThat(sFieldData.multiValued(), equalTo(false));
-
-        assertThat(mFieldData.fieldName(), equalTo("mvalue"));
-        assertThat(mFieldData.multiValued(), equalTo(true));
-
-        // svalue
-        assertThat(sFieldData.hasValue(0), equalTo(true));
-        assertThat(sFieldData.docFieldData(0).isEmpty(), equalTo(false));
-        assertThat(sFieldData.value(0), equalTo((short) 4));
-        assertThat(sFieldData.docFieldData(0).getValue(), equalTo((short) 4));
-        assertThat(sFieldData.values(0).length, equalTo(1));
-        assertThat(sFieldData.docFieldData(0).getValues().length, equalTo(1));
-        assertThat(sFieldData.values(0)[0], equalTo((short) 4));
-        assertThat(sFieldData.docFieldData(0).getValues()[0], equalTo((short) 4));
-
-        assertThat(sFieldData.hasValue(1), equalTo(true));
-        assertThat(sFieldData.value(1), equalTo((short) 3));
-        assertThat(sFieldData.values(1).length, equalTo(1));
-        assertThat(sFieldData.values(1)[0], equalTo((short) 3));
-
-        assertThat(sFieldData.hasValue(2), equalTo(true));
-        assertThat(sFieldData.value(2), equalTo((short) 7));
-        assertThat(sFieldData.values(2).length, equalTo(1));
-        assertThat(sFieldData.values(2)[0], equalTo((short) 7));
-
-        assertThat(sFieldData.hasValue(3), equalTo(false));
-
-        assertThat(sFieldData.hasValue(4), equalTo(true));
-        assertThat(sFieldData.value(4), equalTo((short) 4));
-        assertThat(sFieldData.values(4).length, equalTo(1));
-        assertThat(sFieldData.values(4)[0], equalTo((short) 4));
-
-        // check order is correct
-        final ArrayList<Short> values = new ArrayList<Short>();
-        sFieldData.forEachValue(new ShortFieldData.ValueProc() {
-            @Override
-            public void onValue(short value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo((short) 3));
-        assertThat(values.get(1), equalTo((short) 4));
-        assertThat(values.get(2), equalTo((short) 7));
-
-
-        // mvalue
-        assertThat(mFieldData.hasValue(0), equalTo(true));
-        assertThat(mFieldData.value(0), equalTo((short) 104));
-        assertThat(mFieldData.values(0).length, equalTo(1));
-        assertThat(mFieldData.values(0)[0], equalTo((short) 104));
-
-        assertThat(mFieldData.hasValue(1), equalTo(true));
-        assertThat(mFieldData.value(1), equalTo((short) 104));
-        assertThat(mFieldData.docFieldData(1).getValue(), equalTo((short) 104));
-        assertThat(mFieldData.values(1).length, equalTo(2));
-        assertThat(mFieldData.docFieldData(1).getValues().length, equalTo(2));
-        assertThat(mFieldData.values(1)[0], equalTo((short) 104));
-        assertThat(mFieldData.docFieldData(1).getValues()[0], equalTo((short) 104));
-        assertThat(mFieldData.docFieldData(1).getValues()[1], equalTo((short) 105));
-
-        assertThat(mFieldData.hasValue(2), equalTo(false));
-
-        assertThat(mFieldData.hasValue(3), equalTo(true));
-        assertThat(mFieldData.value(3), equalTo((short) 102));
-        assertThat(mFieldData.values(3).length, equalTo(1));
-        assertThat(mFieldData.values(3)[0], equalTo((short) 102));
-
-        assertThat(mFieldData.hasValue(4), equalTo(false));
-
-        indexWriter.close();
-
-        // check order is correct
-        values.clear();
-        mFieldData.forEachValue(new ShortFieldData.ValueProc() {
-            @Override
-            public void onValue(short value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0), equalTo((short) 102));
-        assertThat(values.get(1), equalTo((short) 104));
-        assertThat(values.get(2), equalTo((short) 105));
-    }
-}
\ No newline at end of file
diff --git a/src/test/java/org/elasticsearch/test/unit/index/field/data/strings/StringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/field/data/strings/StringFieldDataTests.java
deleted file mode 100644
index d266657340e..00000000000
--- a/src/test/java/org/elasticsearch/test/unit/index/field/data/strings/StringFieldDataTests.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.unit.index.field.data.strings;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.field.data.FieldData;
-import org.elasticsearch.index.field.data.strings.StringFieldData;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-public class StringFieldDataTests {
-
-    @Test
-    public void stringFieldDataTests() throws Exception {
-        Directory dir = new RAMDirectory();
-        IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
-
-        Document document = new Document();
-        document.add(new TextField("svalue", "zzz", Field.Store.YES));
-        document.add(new TextField("mvalue", "111", Field.Store.YES));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new TextField("svalue", "xxx", Field.Store.YES));
-        document.add(new TextField("mvalue", "222 333", Field.Store.YES));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new TextField("mvalue", "333 444", Field.Store.YES));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new TextField("svalue", "aaa", Field.Store.YES));
-        indexWriter.addDocument(document);
-
-        document = new Document();
-        document.add(new TextField("svalue", "aaa", Field.Store.YES));
-        indexWriter.addDocument(document);
-
-        AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(indexWriter, false));
-
-        StringFieldData sFieldData = StringFieldData.load(reader, "svalue");
-        StringFieldData mFieldData = StringFieldData.load(reader, "mvalue");
-
-        assertThat(sFieldData.fieldName(), equalTo("svalue"));
-        assertThat(sFieldData.multiValued(), equalTo(false));
-
-        assertThat(mFieldData.fieldName(), equalTo("mvalue"));
-        assertThat(mFieldData.multiValued(), equalTo(true));
-
-        // svalue
-        assertThat(sFieldData.hasValue(0), equalTo(true));
-        assertThat(sFieldData.value(0).utf8ToString(), equalTo("zzz"));
-        assertThat(sFieldData.docFieldData(0).getValue(), equalTo("zzz"));
-        assertThat(sFieldData.values(0).length, equalTo(1));
-        assertThat(sFieldData.docFieldData(0).getValues().length, equalTo(1));
-        assertThat(sFieldData.values(0)[0].utf8ToString(), equalTo("zzz"));
-        assertThat(sFieldData.docFieldData(0).getValues()[0], equalTo("zzz"));
-
-        assertThat(sFieldData.hasValue(1), equalTo(true));
-        assertThat(sFieldData.value(1).utf8ToString(), equalTo("xxx"));
-        assertThat(sFieldData.values(1).length, equalTo(1));
-        assertThat(sFieldData.values(1)[0].utf8ToString(), equalTo("xxx"));
-
-        assertThat(sFieldData.hasValue(2), equalTo(false));
-
-        assertThat(sFieldData.hasValue(3), equalTo(true));
-        assertThat(sFieldData.value(3).utf8ToString(), equalTo("aaa"));
-        assertThat(sFieldData.values(3).length, equalTo(1));
-        assertThat(sFieldData.values(3)[0].utf8ToString(), equalTo("aaa"));
-
-        assertThat(sFieldData.hasValue(4), equalTo(true));
-        assertThat(sFieldData.value(4).utf8ToString(), equalTo("aaa"));
-        assertThat(sFieldData.values(4).length, equalTo(1));
-        assertThat(sFieldData.values(4)[0].utf8ToString(), equalTo("aaa"));
-
-        // check order is correct
-        final ArrayList<BytesRef> values = new ArrayList<BytesRef>();
-        sFieldData.forEachValue(new FieldData.StringValueProc() {
-            @Override
-            public void onValue(BytesRef value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(3));
-
-        assertThat(values.get(0).utf8ToString(), equalTo("aaa"));
-        assertThat(values.get(1).utf8ToString(), equalTo("xxx"));
-        assertThat(values.get(2).utf8ToString(), equalTo("zzz"));
-
-        // mvalue
-        assertThat(mFieldData.hasValue(0), equalTo(true));
-        assertThat(mFieldData.value(0).utf8ToString(), equalTo("111"));
-        assertThat(mFieldData.values(0).length, equalTo(1));
-        assertThat(mFieldData.values(0)[0].utf8ToString(), equalTo("111"));
-
-        assertThat(mFieldData.hasValue(1), equalTo(true));
-        assertThat(mFieldData.value(1).utf8ToString(), equalTo("222"));
-        assertThat(mFieldData.values(1).length, equalTo(2));
-        assertThat(mFieldData.values(1)[0].utf8ToString(), equalTo("222"));
-        assertThat(mFieldData.values(1)[1].utf8ToString(), equalTo("333"));
-
-        assertThat(mFieldData.hasValue(2), equalTo(true));
-        assertThat(mFieldData.value(2).utf8ToString(), equalTo("333"));
-        assertThat(mFieldData.values(2).length, equalTo(2));
-        assertThat(mFieldData.values(2)[0].utf8ToString(), equalTo("333"));
-        assertThat(mFieldData.values(2)[1].utf8ToString(), equalTo("444"));
-
-        assertThat(mFieldData.hasValue(3), equalTo(false));
-
-        assertThat(mFieldData.hasValue(4), equalTo(false));
-
-        values.clear();
-        mFieldData.forEachValue(new FieldData.StringValueProc() {
-            @Override
-            public void onValue(BytesRef value) {
-                values.add(value);
-            }
-        });
-        assertThat(values.size(), equalTo(4));
-
-        assertThat(values.get(0).utf8ToString(), equalTo("111"));
-        assertThat(values.get(1).utf8ToString(), equalTo("222"));
-        assertThat(values.get(2).utf8ToString(), equalTo("333"));
-        assertThat(values.get(3).utf8ToString(), equalTo("444"));
-
-        indexWriter.close();
-    }
-}
diff --git a/src/test/java/org/elasticsearch/test/unit/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java b/src/test/java/org/elasticsearch/test/unit/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java
index 6a0a60d4045..6c851c80a80 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/mapper/dynamictemplate/pathmatch/PathMatchDynamicTemplateTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.test.unit.index.mapper.dynamictemplate.pathmatch;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.index.field.data.strings.StringFieldDataType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.test.unit.index.mapper.MapperTests;
@@ -32,7 +31,6 @@ import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
 import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
 
 /**
  *
@@ -75,6 +73,5 @@ public class PathMatchDynamicTemplateTests {
 
         fieldMappers = docMapper.mappers().fullName("obj3.obj4.prop1");
         assertThat(fieldMappers.mappers().size(), equalTo(1));
-        assertThat(fieldMappers.mapper().fieldDataType(), instanceOf(StringFieldDataType.class));
     }
 }