From c43d0d17468a13aaee6030d4e9f11b2def6095e1 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 15 Aug 2013 19:24:02 +0200
Subject: [PATCH] Added infrastructure to figure out the number of unique
 values for a field on the atomic level and the highest number of atomic field
 values for all segments. This can use as a heuristic for initializing data
 structures.

Also moved the load method from concrete classes to AbstractIndexFieldData class.
---
 .../fielddata/AbstractIndexFieldData.java     | 35 +++++++++++++++++++
 .../index/fielddata/AtomicFieldData.java      |  5 +++
 .../index/fielddata/IndexFieldData.java       |  5 +++
 .../plain/AbstractBytesIndexFieldData.java    | 19 +++-------
 .../plain/DoubleArrayAtomicFieldData.java     | 28 +++++++++++++--
 .../plain/DoubleArrayIndexFieldData.java      | 18 ++--------
 .../plain/FSTBytesAtomicFieldData.java        |  9 ++++-
 .../plain/FloatArrayAtomicFieldData.java      | 28 +++++++++++++--
 .../plain/FloatArrayIndexFieldData.java       | 18 ++--------
 .../GeoPointDoubleArrayAtomicFieldData.java   | 28 +++++++++++++--
 .../GeoPointDoubleArrayIndexFieldData.java    | 18 ++--------
 .../plain/PackedArrayAtomicFieldData.java     | 28 +++++++++++++--
 .../plain/PackedArrayIndexFieldData.java      | 18 ++--------
 .../plain/PagedBytesAtomicFieldData.java      | 10 ++++++
 .../AbstractStringFieldDataTests.java         | 13 ++++++-
 15 files changed, 191 insertions(+), 89 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
index 77af03f40cd..3deaad3e440 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
@@ -1,17 +1,22 @@
 package org.elasticsearch.index.fielddata;
 
+import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
+import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.settings.IndexSettings;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 /**
  */
 public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
 
     private final FieldMapper.Names fieldNames;
+    private final AtomicLong highestUniqueValuesCount = new AtomicLong();
     protected final FieldDataType fieldDataType;
     protected final IndexFieldDataCache cache;
 
@@ -36,4 +41,34 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
     public void clear(IndexReader reader) {
         cache.clear(reader);
     }
+
+    @Override
+    public long getHighestNumberOfSeenUniqueValues() {
+        return highestUniqueValuesCount.get();
+    }
+
+    @Override
+    public final FD load(AtomicReaderContext context) {
+        try {
+            FD fd = cache.load(context, this);
+            updateHighestSeenValuesCount(fd.getNumberUniqueValues());
+            return fd;
+        } catch (Throwable e) {
+            if (e instanceof ElasticSearchException) {
+                throw (ElasticSearchException) e;
+            } else {
+                throw new ElasticSearchException(e.getMessage(), e);
+            }
+        }
+    }
+
+    private void updateHighestSeenValuesCount(long newValuesCount) {
+        long current;
+        do {
+            if ((current = highestUniqueValuesCount.get()) >= newValuesCount) {
+                break;
+            }
+        } while (!highestUniqueValuesCount.compareAndSet(current, newValuesCount));
+    }
+
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
index d15497bd0e9..254e94b5d9c 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
@@ -39,6 +39,11 @@ public interface AtomicFieldData<Script extends ScriptDocValues> {
      */
     int getNumDocs();
 
+    /**
+     * The number of unique values in this atomic field data.
+     */
+    long getNumberUniqueValues();
+
     /**
      * Size (in bytes) of memory used by this field data.
      */
diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index 8d65fbb5489..6b16cff59a8 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -79,6 +79,11 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
 
     void clear(IndexReader reader);
 
+    /**
+     * Returns the highest ever seen uniqiue values in an atomic reader.
+     */
+    long getHighestNumberOfSeenUniqueValues();
+
     // we need this extended source we we have custom comparators to reuse our field data
     // in this case, we need to reduce type that will be used when search results are reduced
     // on another node (we don't have the custom source them...)
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java
index 1a5f9f96e4f..282780b6fe9 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java
@@ -18,11 +18,13 @@
  */
 package org.elasticsearch.index.fielddata.plain;
 
-import org.apache.lucene.index.*;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.FilteredTermsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.UnicodeUtil;
-import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -55,19 +57,6 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
         return true;
     }
 
-    @Override
-    public FD load(AtomicReaderContext context) {
-        try {
-            return cache.load(context, this);
-        } catch (Throwable e) {
-            if (e instanceof ElasticSearchException) {
-                throw (ElasticSearchException) e;
-            } else {
-                throw new ElasticSearchException(e.getMessage(), e);
-            }
-        }
-    }
-
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, SortMode sortMode) {
         // TODO support "missingValue" for sortMissingValue options here...
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
index 5476244e3ab..764faee691e 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
@@ -77,6 +77,11 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return 0;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             return 0;
@@ -114,6 +119,11 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData
             return true;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return ordinals.getNumOrds();
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
@@ -173,11 +183,13 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData
 
         private final BigDoubleArrayList values;
         private final FixedBitSet set;
+        private final long numOrds;
 
-        public SingleFixedSet(BigDoubleArrayList values, int numDocs, FixedBitSet set) {
+        public SingleFixedSet(BigDoubleArrayList values, int numDocs, FixedBitSet set, long numOrds) {
             super(numDocs);
             this.values = values;
             this.set = set;
+            this.numOrds = numOrds;
         }
 
         @Override
@@ -190,6 +202,11 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrds;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
@@ -260,14 +277,16 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData
     public static class Single extends DoubleArrayAtomicFieldData {
 
         private final BigDoubleArrayList values;
+        private final long numOrds;
 
         /**
          * Note, here, we assume that there is no offset by 1 from docId, so position 0
          * is the value for docId 0.
          */
-        public Single(BigDoubleArrayList values, int numDocs) {
+        public Single(BigDoubleArrayList values, int numDocs, long numOrds) {
             super(numDocs);
             this.values = values;
+            this.numOrds = numOrds;
         }
 
         @Override
@@ -280,6 +299,11 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrds;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
index f4afcc3e3ab..1c099026842 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
@@ -23,7 +23,6 @@ import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.*;
-import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigDoubleArrayList;
@@ -65,19 +64,6 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
         return false;
     }
 
-    @Override
-    public DoubleArrayAtomicFieldData load(AtomicReaderContext context) {
-        try {
-            return cache.load(context, this);
-        } catch (Throwable e) {
-            if (e instanceof ElasticSearchException) {
-                throw (ElasticSearchException) e;
-            } else {
-                throw new ElasticSearchException(e.getMessage(), e);
-            }
-        }
-    }
-
     @Override
     public DoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
 
@@ -118,9 +104,9 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
                 }
                 assert sValues.size() == maxDoc;
                 if (set == null) {
-                    return new DoubleArrayAtomicFieldData.Single(sValues, maxDoc);
+                    return new DoubleArrayAtomicFieldData.Single(sValues, maxDoc, ordinals.getNumOrds());
                 } else {
-                    return new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, maxDoc, set);
+                    return new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, maxDoc, set, ordinals.getNumOrds());
                 }
             } else {
                 return new DoubleArrayAtomicFieldData.WithOrdinals(
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
index 816b2587e41..de94ae3ddb9 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
@@ -21,9 +21,11 @@ package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.fst.*;
+import org.apache.lucene.util.fst.BytesRefFSTEnum;
+import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.FST.Arc;
 import org.apache.lucene.util.fst.FST.BytesReader;
+import org.apache.lucene.util.fst.Util;
 import org.elasticsearch.common.util.BigIntArray;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
@@ -68,6 +70,11 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
         return ordinals.getNumDocs();
     }
 
+    @Override
+    public long getNumberUniqueValues() {
+        return ordinals.getNumOrds();
+    }
+
     @Override
     public boolean isValuesOrdered() {
         return true;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
index d90d36005ba..52a1795cb4f 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
@@ -72,6 +72,11 @@ public abstract class FloatArrayAtomicFieldData extends AtomicNumericFieldData {
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return 0;
+        }
+
         @Override
         public boolean isValuesOrdered() {
             return false;
@@ -114,6 +119,11 @@ public abstract class FloatArrayAtomicFieldData extends AtomicNumericFieldData {
             return true;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return ordinals.getNumOrds();
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
@@ -171,11 +181,13 @@ public abstract class FloatArrayAtomicFieldData extends AtomicNumericFieldData {
 
         private final BigFloatArrayList values;
         private final FixedBitSet set;
+        private final long numOrd;
 
-        public SingleFixedSet(BigFloatArrayList values, int numDocs, FixedBitSet set) {
+        public SingleFixedSet(BigFloatArrayList values, int numDocs, FixedBitSet set, long numOrd) {
             super(numDocs);
             this.values = values;
             this.set = set;
+            this.numOrd = numOrd;
         }
 
         @Override
@@ -188,6 +200,11 @@ public abstract class FloatArrayAtomicFieldData extends AtomicNumericFieldData {
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrd;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
@@ -260,14 +277,16 @@ public abstract class FloatArrayAtomicFieldData extends AtomicNumericFieldData {
     public static class Single extends FloatArrayAtomicFieldData {
 
         private final BigFloatArrayList values;
+        private final long numOrd;
 
         /**
          * Note, here, we assume that there is no offset by 1 from docId, so position 0
          * is the value for docId 0.
          */
-        public Single(BigFloatArrayList values, int numDocs) {
+        public Single(BigFloatArrayList values, int numDocs, long numOrd) {
             super(numDocs);
             this.values = values;
+            this.numOrd = numOrd;
         }
 
         @Override
@@ -280,6 +299,11 @@ public abstract class FloatArrayAtomicFieldData extends AtomicNumericFieldData {
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrd;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
index 5668ccc4937..31f3db43183 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
@@ -23,7 +23,6 @@ import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.*;
-import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigFloatArrayList;
@@ -65,19 +64,6 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
         return false;
     }
 
-    @Override
-    public FloatArrayAtomicFieldData load(AtomicReaderContext context) {
-        try {
-            return cache.load(context, this);
-        } catch (Throwable e) {
-            if (e instanceof ElasticSearchException) {
-                throw (ElasticSearchException) e;
-            } else {
-                throw new ElasticSearchException(e.getMessage(), e);
-            }
-        }
-    }
-
     @Override
     public FloatArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
@@ -118,9 +104,9 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
                 }
                 assert sValues.size() == maxDoc;
                 if (set == null) {
-                    return new FloatArrayAtomicFieldData.Single(sValues, maxDoc);
+                    return new FloatArrayAtomicFieldData.Single(sValues, maxDoc, ordinals.getNumOrds());
                 } else {
-                    return new FloatArrayAtomicFieldData.SingleFixedSet(sValues, maxDoc, set);
+                    return new FloatArrayAtomicFieldData.SingleFixedSet(sValues, maxDoc, set, ordinals.getNumOrds());
                 }
             } else {
                 return new FloatArrayAtomicFieldData.WithOrdinals(
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
index ded89f692b8..a260df79005 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
@@ -75,6 +75,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return 0;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             return 0;
@@ -118,6 +123,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
             return true;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return ordinals.getNumOrds();
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
@@ -253,12 +263,14 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
 
         private final BigDoubleArrayList lon, lat;
         private final FixedBitSet set;
+        private final long numOrds;
 
-        public SingleFixedSet(BigDoubleArrayList lon, BigDoubleArrayList lat, int numDocs, FixedBitSet set) {
+        public SingleFixedSet(BigDoubleArrayList lon, BigDoubleArrayList lat, int numDocs, FixedBitSet set, long numOrds) {
             super(numDocs);
             this.lon = lon;
             this.lat = lat;
             this.set = set;
+            this.numOrds = numOrds;
         }
 
         @Override
@@ -271,6 +283,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrds;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
@@ -351,11 +368,13 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
     public static class Single extends GeoPointDoubleArrayAtomicFieldData {
 
         private final BigDoubleArrayList lon, lat;
+        private final long numOrds;
 
-        public Single(BigDoubleArrayList lon, BigDoubleArrayList lat, int numDocs) {
+        public Single(BigDoubleArrayList lon, BigDoubleArrayList lat, int numDocs, long numOrds) {
             super(numDocs);
             this.lon = lon;
             this.lat = lat;
+            this.numOrds = numOrds;
         }
 
         @Override
@@ -368,6 +387,11 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrds;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
index 448b0c5fca4..b6dc7c78f20 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java
@@ -23,7 +23,6 @@ import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.*;
-import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
@@ -60,19 +59,6 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<Ge
         return false;
     }
 
-    @Override
-    public GeoPointDoubleArrayAtomicFieldData load(AtomicReaderContext context) {
-        try {
-            return cache.load(context, this);
-        } catch (Throwable e) {
-            if (e instanceof ElasticSearchException) {
-                throw (ElasticSearchException) e;
-            } else {
-                throw new ElasticSearchException(e.getMessage(), e);
-            }
-        }
-    }
-
     @Override
     public GeoPointDoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
@@ -119,9 +105,9 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<Ge
                 }
                 FixedBitSet set = builder.buildDocsWithValuesSet();
                 if (set == null) {
-                    return new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat, reader.maxDoc());
+                    return new GeoPointDoubleArrayAtomicFieldData.Single(sLon, sLat, reader.maxDoc(), ordinals.getNumOrds());
                 } else {
-                    return new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, reader.maxDoc(), set);
+                    return new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(sLon, sLat, reader.maxDoc(), set, ordinals.getNumOrds());
                 }
             } else {
                 return new GeoPointDoubleArrayAtomicFieldData.WithOrdinals(
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java
index c1a7e233836..efcec015b41 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java
@@ -83,6 +83,11 @@ public abstract class PackedArrayAtomicFieldData extends AtomicNumericFieldData
             return 0;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return 0;
+        }
+
         @Override
         public BytesValues getBytesValues() {
             return BytesValues.EMPTY;
@@ -123,6 +128,11 @@ public abstract class PackedArrayAtomicFieldData extends AtomicNumericFieldData
             return size;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return ordinals.getNumOrds();
+        }
+
         @Override
         public LongValues getLongValues() {
             return new LongValues(values, ordinals.ordinals());
@@ -175,12 +185,14 @@ public abstract class PackedArrayAtomicFieldData extends AtomicNumericFieldData
         private final PackedInts.Mutable values;
         private final long minValue;
         private final long missingValue;
+        private final long numOrds;
 
-        public SingleSparse(PackedInts.Mutable values, long minValue, int numDocs, long missingValue) {
+        public SingleSparse(PackedInts.Mutable values, long minValue, int numDocs, long missingValue, long numOrds) {
             super(numDocs);
             this.values = values;
             this.minValue = minValue;
             this.missingValue = missingValue;
+            this.numOrds = numOrds;
         }
 
         @Override
@@ -193,6 +205,11 @@ public abstract class PackedArrayAtomicFieldData extends AtomicNumericFieldData
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrds;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
@@ -270,15 +287,17 @@ public abstract class PackedArrayAtomicFieldData extends AtomicNumericFieldData
 
         private final PackedInts.Mutable values;
         private final long minValue;
+        private final long numOrds;
 
         /**
          * Note, here, we assume that there is no offset by 1 from docId, so position 0
          * is the value for docId 0.
          */
-        public Single(PackedInts.Mutable values, long minValue, int numDocs) {
+        public Single(PackedInts.Mutable values, long minValue, int numDocs, long numOrds) {
             super(numDocs);
             this.values = values;
             this.minValue = minValue;
+            this.numOrds = numOrds;
         }
 
         @Override
@@ -291,6 +310,11 @@ public abstract class PackedArrayAtomicFieldData extends AtomicNumericFieldData
             return false;
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return numOrds;
+        }
+
         @Override
         public long getMemorySizeInBytes() {
             if (size == -1) {
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java
index f0e34dad505..1bb33bc56f0 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java
@@ -29,7 +29,6 @@ import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
 import org.apache.lucene.util.packed.PackedInts;
-import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -85,19 +84,6 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
         return false;
     }
 
-    @Override
-    public AtomicNumericFieldData load(AtomicReaderContext context) {
-        try {
-            return cache.load(context, this);
-        } catch (Throwable e) {
-            if (e instanceof ElasticSearchException) {
-                throw (ElasticSearchException) e;
-            } else {
-                throw new ElasticSearchException(e.getMessage(), e);
-            }
-        }
-    }
-
     @Override
     public AtomicNumericFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
@@ -184,9 +170,9 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
                     }
                 }
                 if (set == null) {
-                    return new PackedArrayAtomicFieldData.Single(sValues, minValue, reader.maxDoc());
+                    return new PackedArrayAtomicFieldData.Single(sValues, minValue, reader.maxDoc(), ordinals.getNumOrds());
                 } else {
-                    return new PackedArrayAtomicFieldData.SingleSparse(sValues, minValue, reader.maxDoc(), missingValue);
+                    return new PackedArrayAtomicFieldData.SingleSparse(sValues, minValue, reader.maxDoc(), missingValue, ordinals.getNumOrds());
                 }
             } else {
                 return new PackedArrayAtomicFieldData.WithOrdinals(values, reader.maxDoc(), build);
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
index 16b7c37b61e..4537ede4142 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
@@ -68,6 +68,11 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
         return ordinals.getNumDocs();
     }
 
+    @Override
+    public long getNumberUniqueValues() {
+        return ordinals.getNumOrds();
+    }
+
     @Override
     public boolean isValuesOrdered() {
         return true;
@@ -265,6 +270,11 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
             return ordinals.getNumDocs();
         }
 
+        @Override
+        public long getNumberUniqueValues() {
+            return 0;
+        }
+
         @Override
         public boolean isValuesOrdered() {
             return true;
diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java
index 8676d9221e1..085df65c38a 100644
--- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java
@@ -34,7 +34,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
 import org.junit.Test;
 
-import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
 /**
@@ -103,8 +102,10 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
         IndexFieldData indexFieldData = getForField("value");
         AtomicReaderContext readerContext = refreshReader();
         AtomicFieldData fieldData = indexFieldData.load(readerContext);
+        assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
         BytesValues values = fieldData.getBytesValues();
         assertThat(fieldData.getNumDocs(), equalTo(2));
+        assertThat(fieldData.getNumberUniqueValues(), equalTo(2l));
         for (int i = 0; i < fieldData.getNumDocs(); ++i) {
             assertThat(values.hasValue(i), equalTo(true));
         }
@@ -117,6 +118,8 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
         AtomicReaderContext readerContext = refreshReader();
         AtomicFieldData fieldData = indexFieldData.load(readerContext);
         assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
+        assertThat(fieldData.getNumberUniqueValues(), equalTo(3l));
+        assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
 
@@ -211,6 +214,8 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
         AtomicFieldData fieldData = indexFieldData.load(refreshReader());
         assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
 
+        assertThat(fieldData.getNumberUniqueValues(), equalTo(2l));
+        assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
         assertThat(fieldData.getNumDocs(), equalTo(3));
 
         BytesValues bytesValues = fieldData
@@ -291,6 +296,8 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
         assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
+        assertThat(fieldData.getNumberUniqueValues(), equalTo(4l));
+        assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
 
         BytesValues bytesValues = fieldData.getBytesValues();
 
@@ -379,6 +386,8 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
         assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
+        assertThat(fieldData.getNumberUniqueValues(), equalTo(3l));
+        assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
 
         BytesValues bytesValues = fieldData.getBytesValues();
 
@@ -440,6 +449,8 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
         assertThat(fieldData.getMemorySizeInBytes(), greaterThanOrEqualTo(0l));
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
+        assertThat(fieldData.getNumberUniqueValues(), equalTo(0l));
+        assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), equalTo(0l));
 
         BytesValues bytesValues = fieldData.getBytesValues();