From 454dc53483c69a59090bf1c50bb2708e480c5858 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Wed, 30 Oct 2013 11:23:38 +0100
Subject: [PATCH] Add back hashed BytesValues optimization that got lost in a
 previous commit.

Some FieldData consumers require hash values per byte. We provide an optimization
that allows to cache the hashes internally if the consumer knows that they are needed
this optimization got lost in a previous commit. This commit adds them back and folds
the dedicated method into AtomicFieldData#getBytesValues(true|false)
---
 .../AbstractAtomicNumericFieldData.java       |  7 +--
 .../index/fielddata/AtomicFieldData.java      | 16 ++---
 .../fielddata/AtomicGeoPointFieldData.java    |  7 +--
 .../BytesRefOrdValComparator.java             |  2 +-
 .../BytesRefValComparator.java                |  2 +-
 .../plain/BinaryDVAtomicFieldData.java        | 13 ++--
 .../plain/DoubleArrayAtomicFieldData.java     |  2 +-
 .../plain/FSTBytesAtomicFieldData.java        | 61 +++++++++----------
 .../plain/FloatArrayAtomicFieldData.java      |  2 +-
 .../GeoPointDoubleArrayAtomicFieldData.java   |  2 +-
 .../plain/PackedArrayAtomicFieldData.java     |  2 +-
 .../plain/PagedBytesAtomicFieldData.java      | 19 +++---
 .../plain/SortedSetDVAtomicFieldData.java     |  2 +-
 .../SortedSetDVBytesAtomicFieldData.java      |  2 +-
 .../SortedSetDVNumericAtomicFieldData.java    | 14 ++---
 .../percolator/QueriesLoaderCollector.java    |  4 +-
 .../percolator/PercolatorService.java         |  2 +-
 .../percolator/QueryCollector.java            |  2 +-
 .../FieldsTermsStringFacetExecutor.java       |  2 +-
 .../strings/TermsStringFacetExecutor.java     |  6 +-
 .../TermsStringOrdinalsFacetExecutor.java     |  2 +-
 .../TermsStatsStringFacetExecutor.java        |  2 +-
 .../fielddata/AbstractFieldDataImplTests.java | 22 +++----
 .../index/fielddata/DuelFieldDataTests.java   |  4 +-
 .../index/fielddata/FilterFieldDataTest.java  | 19 +++---
 25 files changed, 93 insertions(+), 125 deletions(-)

diff --git a/src/main/java/org/elasticsearch/index/fielddata/AbstractAtomicNumericFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AbstractAtomicNumericFieldData.java
index a182f7023bb..1bf5fb03fa6 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AbstractAtomicNumericFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AbstractAtomicNumericFieldData.java
@@ -42,7 +42,7 @@ public abstract class AbstractAtomicNumericFieldData implements AtomicNumericFie
     }
 
     @Override
-    public BytesValues getBytesValues() {
+    public BytesValues getBytesValues(boolean needsHashes) {
         if (isFloat) {
             final DoubleValues values = getDoubleValues();
             return new BytesValues(values.isMultiValued()) {
@@ -106,9 +106,4 @@ public abstract class AbstractAtomicNumericFieldData implements AtomicNumericFie
             };
         }
     }
-
-    @Override
-    public BytesValues getHashedBytesValues() {
-        return getBytesValues();
-    }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
index 3cf0c74be0b..07df8caef0d 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
@@ -53,12 +53,14 @@ public interface AtomicFieldData<Script extends ScriptDocValues> {
 
     /**
      * Use a non thread safe (lightweight) view of the values as bytes.
+     *
+     * @param needsHashes if <code>true</code> the implementation will use pre-build hashes if
+     *                    {@link org.elasticsearch.index.fielddata.BytesValues#currentValueHash()} is used. if no hashes
+     *                    are used <code>false</code> should be passed instead.
+     *
      */
-    BytesValues getBytesValues();
+    BytesValues getBytesValues(boolean needsHashes);
     
-    
-    BytesValues getHashedBytesValues();
-
     /**
      * Returns a "scripting" based values.
      */
@@ -73,10 +75,8 @@ public interface AtomicFieldData<Script extends ScriptDocValues> {
 
         /**
          * Use a non thread safe (lightweight) view of the values as bytes.
+         * @param needsHashes
          */
-        BytesValues.WithOrdinals getBytesValues();
-        
-        
-        BytesValues.WithOrdinals getHashedBytesValues();
+        BytesValues.WithOrdinals getBytesValues(boolean needsHashes);
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
index b36d8f474a6..dff3cfe9b4a 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
@@ -30,7 +30,7 @@ public abstract class AtomicGeoPointFieldData<Script extends ScriptDocValues> im
     public abstract GeoPointValues getGeoPointValues();
 
     @Override
-    public BytesValues getBytesValues() {
+    public BytesValues getBytesValues(boolean needsHashes) {
         final GeoPointValues values = getGeoPointValues();
         return new BytesValues(values.isMultiValued()) {
 
@@ -67,9 +67,4 @@ public abstract class AtomicGeoPointFieldData<Script extends ScriptDocValues> im
         };
     }
 
-    @Override
-    public BytesValues getHashedBytesValues() {
-        return getBytesValues();
-    }
-
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
index 4b595914caa..b762d0cebf5 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
@@ -271,7 +271,7 @@ public final class BytesRefOrdValComparator extends NestedWrappableComparator<By
 
     @Override
     public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-        termsIndex = indexFieldData.load(context).getBytesValues();
+        termsIndex = indexFieldData.load(context).getBytesValues(false);
         assert termsIndex.ordinals() != null && termsIndex.ordinals().ordinals() != null;
         if (missingValue == null) {
             missingOrd = Ordinals.MISSING_ORDINAL;
diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
index 93400d86248..2888c15ce9f 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
@@ -82,7 +82,7 @@ public final class BytesRefValComparator extends NestedWrappableComparator<Bytes
 
     @Override
     public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
-        docTerms = indexFieldData.load(context).getBytesValues();
+        docTerms = indexFieldData.load(context).getBytesValues(false);
         if (docTerms.isMultiValued()) {
             docTerms = new MultiValuedBytesWrapper(docTerms, sortMode);
         }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java
index 497cb466de9..1fc4a0dd3d5 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java
@@ -70,7 +70,9 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
     }
 
     @Override
-    public BytesValues getBytesValues() {
+    public BytesValues getBytesValues(boolean needsHashes) {
+        // if you want hashes to be cached, you should rather store them on disk alongside the values rather than loading them into memory
+        // here - not supported for now, and probably not useful since this field data only applies to _id and _uid?
         final BinaryDocValues values;
         final Bits docsWithField;
         try {
@@ -108,16 +110,9 @@ public class BinaryDVAtomicFieldData implements AtomicFieldData<ScriptDocValues.
         };
     }
 
-    @Override
-    public BytesValues getHashedBytesValues() {
-        // if you want hashes to be cached, you should rather store them on disk alongside the values rather than loading them into memory
-        // here - not supported for now, and probably not useful since this field data only applies to _id and _uid?
-        return getBytesValues();
-    }
-
     @Override
     public Strings getScriptValues() {
-        return new ScriptDocValues.Strings(getBytesValues());
+        return new ScriptDocValues.Strings(getBytesValues(false));
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
index 8574b24e863..a73bff7b002 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
@@ -88,7 +88,7 @@ public abstract class DoubleArrayAtomicFieldData extends AbstractAtomicNumericFi
         }
 
         @Override
-        public BytesValues getBytesValues() {
+        public BytesValues getBytesValues(boolean needsHashes) {
             return BytesValues.EMPTY;
         }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
index 9fe847bcafe..ecbded35f78 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
@@ -90,42 +90,42 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
     }
 
     @Override
-    public BytesValues.WithOrdinals getBytesValues() {
+    public BytesValues.WithOrdinals getBytesValues(boolean needsHashes) {
         assert fst != null;
-        return new BytesValues(fst, ordinals.ordinals());
+        if (needsHashes) {
+            if (hashes == null) {
+                BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst);
+                BigIntArray hashes = new BigIntArray(ordinals.getMaxOrd());
+                // we don't store an ord 0 in the FST since we could have an empty string in there and FST don't support
+                // empty strings twice. ie. them merge fails for long output.
+                hashes.set(0, new BytesRef().hashCode());
+                try {
+                    for (long i = 1, maxOrd = ordinals.getMaxOrd(); i < maxOrd; ++i) {
+                        hashes.set(i, fstEnum.next().input.hashCode());
+                    }
+                    assert fstEnum.next() == null;
+                } catch (IOException e) {
+                    // Don't use new "AssertionError("Cannot happen", e)" directly as this is a Java 1.7-only API
+                    final AssertionError error = new AssertionError("Cannot happen");
+                    error.initCause(e);
+                    throw error;
+                }
+                this.hashes = hashes;
+            }
+            return new HashedBytesValues(fst, ordinals.ordinals(), hashes);
+        } else {
+            return new BytesValues(fst, ordinals.ordinals());
+        }
     }
 
 
     @Override
     public ScriptDocValues.Strings getScriptValues() {
         assert fst != null;
-        return new ScriptDocValues.Strings(getBytesValues());
+        return new ScriptDocValues.Strings(getBytesValues(false));
     }
 
-    @Override
-    public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getHashedBytesValues() {
-        assert fst != null;
-        if (hashes == null) {
-            BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst);
-            BigIntArray hashes = new BigIntArray(ordinals.getMaxOrd());
-            // we don't store an ord 0 in the FST since we could have an empty string in there and FST don't support
-            // empty strings twice. ie. them merge fails for long output.
-            hashes.set(0, new BytesRef().hashCode());
-            try {
-                for (long i = 1, maxOrd = ordinals.getMaxOrd(); i < maxOrd; ++i) {
-                    hashes.set(i, fstEnum.next().input.hashCode());
-                }
-                assert fstEnum.next() == null;
-            } catch (IOException e) {
-                // Don't use new "AssertionError("Cannot happen", e)" directly as this is a Java 1.7-only API
-                final AssertionError error = new AssertionError("Cannot happen");
-                error.initCause(e);
-                throw error;
-            }
-            this.hashes = hashes;
-        }
-        return new HashedBytesValues(fst, ordinals.ordinals(), hashes);
-    }
+
 
     static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
 
@@ -201,7 +201,7 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
         }
 
         @Override
-        public BytesValues.WithOrdinals getBytesValues() {
+        public BytesValues.WithOrdinals getBytesValues(boolean needsHashes) {
             return new EmptyByteValuesWithOrdinals(ordinals.ordinals());
         }
 
@@ -209,11 +209,6 @@ public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<Scr
         public ScriptDocValues.Strings getScriptValues() {
             return ScriptDocValues.EMPTY_STRINGS;
         }
-
-        @Override
-        public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getHashedBytesValues() {
-            return getBytesValues();
-        }
     }
 
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
index 99545b00e18..63d368dd1ca 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
@@ -88,7 +88,7 @@ public abstract class FloatArrayAtomicFieldData extends AbstractAtomicNumericFie
         }
 
         @Override
-        public BytesValues getBytesValues() {
+        public BytesValues getBytesValues(boolean needsHashes) {
             return BytesValues.EMPTY;
         }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
index 625a10260e6..ca4956e08c5 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
@@ -86,7 +86,7 @@ public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointF
         }
 
         @Override
-        public BytesValues getBytesValues() {
+        public BytesValues getBytesValues(boolean needsHashes) {
             return BytesValues.EMPTY;
         }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java
index c3650fc5787..15508ac24ce 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayAtomicFieldData.java
@@ -89,7 +89,7 @@ public abstract class PackedArrayAtomicFieldData extends AbstractAtomicNumericFi
         }
 
         @Override
-        public BytesValues getBytesValues() {
+        public BytesValues getBytesValues(boolean needsHashes) {
             return BytesValues.EMPTY;
         }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
index 03d1aea9a7c..488aecd5246 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
@@ -106,19 +106,18 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
     }
 
     @Override
-    public BytesValues.WithOrdinals getBytesValues() {
-        return new BytesValues(bytes, termOrdToBytesOffset, ordinals.ordinals());
-    }
-
-    @Override
-    public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getHashedBytesValues() {
-        final BigIntArray hashes = getHashes();
-        return new BytesValues.HashedBytesValues(hashes, bytes, termOrdToBytesOffset, ordinals.ordinals());
+    public BytesValues.WithOrdinals getBytesValues(boolean needsHashes) {
+        if (needsHashes) {
+            final BigIntArray hashes = getHashes();
+            return new BytesValues.HashedBytesValues(hashes, bytes, termOrdToBytesOffset, ordinals.ordinals());
+        } else {
+            return new BytesValues(bytes, termOrdToBytesOffset, ordinals.ordinals());
+        }
     }
 
     @Override
     public ScriptDocValues.Strings getScriptValues() {
-        return new ScriptDocValues.Strings(getBytesValues());
+        return new ScriptDocValues.Strings(getBytesValues(false));
     }
 
     static class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
@@ -221,7 +220,7 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
         }
 
         @Override
-        public BytesValues.WithOrdinals getBytesValues() {
+        public BytesValues.WithOrdinals getBytesValues(boolean needsHashes) {
             return new EmptyByteValuesWithOrdinals(ordinals.ordinals());
         }
 
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVAtomicFieldData.java
index 58257facae6..a86d0f9447c 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVAtomicFieldData.java
@@ -74,7 +74,7 @@ abstract class SortedSetDVAtomicFieldData {
         // no-op
     }
 
-    public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getBytesValues() {
+    public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getBytesValues(boolean needsHashes) {
         final SortedSetDocValues values = getValuesNoException(reader, field);
         return new SortedSetValues(reader, field, values);
     }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
index ee6ba2704d8..1b6fc7e6778 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java
@@ -44,6 +44,6 @@ public final class SortedSetDVBytesAtomicFieldData extends SortedSetDVAtomicFiel
 
     @Override
     public Strings getScriptValues() {
-        return new ScriptDocValues.Strings(getBytesValues());
+        return new ScriptDocValues.Strings(getBytesValues(false));
     }
 }
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVNumericAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVNumericAtomicFieldData.java
index dba3ba7b9c0..046b5947d9e 100644
--- a/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVNumericAtomicFieldData.java
+++ b/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVNumericAtomicFieldData.java
@@ -50,7 +50,7 @@ public class SortedSetDVNumericAtomicFieldData extends SortedSetDVAtomicFieldDat
 
     @Override
     public LongValues getLongValues() {
-        final BytesValues.WithOrdinals values = super.getBytesValues();
+        final BytesValues.WithOrdinals values = super.getBytesValues(false);
         return new LongValues.WithOrdinals(values.ordinals()) {
             @Override
             public long getValueByOrd(long ord) {
@@ -62,7 +62,7 @@ public class SortedSetDVNumericAtomicFieldData extends SortedSetDVAtomicFieldDat
 
     @Override
     public DoubleValues getDoubleValues() {
-        final BytesValues.WithOrdinals values = super.getBytesValues();
+        final BytesValues.WithOrdinals values = super.getBytesValues(false);
         return new DoubleValues.WithOrdinals(values.ordinals()) {
             @Override
             public double getValueByOrd(long ord) {
@@ -73,8 +73,8 @@ public class SortedSetDVNumericAtomicFieldData extends SortedSetDVAtomicFieldDat
     }
 
     @Override
-    public BytesValues.WithOrdinals getBytesValues() {
-        final BytesValues.WithOrdinals values = super.getBytesValues();
+    public BytesValues.WithOrdinals getBytesValues(boolean needsHashes) {
+        final BytesValues.WithOrdinals values = super.getBytesValues(needsHashes);
         return new BytesValues.WithOrdinals(values.ordinals()) {
             final BytesRef spare = new BytesRef(16);
             private BytesRef convert(BytesRef input, BytesRef output) {
@@ -95,10 +95,4 @@ public class SortedSetDVNumericAtomicFieldData extends SortedSetDVAtomicFieldDat
             }
         };
     }
-
-    @Override
-    public BytesValues.WithOrdinals getHashedBytesValues() {
-        return getBytesValues();
-    }
-
 }
diff --git a/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java b/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
index 43fcc7ec761..f4fa16cdc7a 100644
--- a/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
+++ b/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
@@ -61,7 +61,7 @@ final class QueriesLoaderCollector extends Collector {
                 // id is only used for logging, if we fail we log the id in the catch statement
                 final Query parseQuery = percolator.parsePercolatorDocument(null, fieldsVisitor.source());
                 if (parseQuery != null) {
-                    queries.put(new HashedBytesRef(idValues.copyShared()), parseQuery);
+                    queries.put(new HashedBytesRef(idValues.copyShared(), idValues.currentValueHash()), parseQuery);
                 } else {
                     logger.warn("failed to add query [{}] - parser returned null", id);
                 }
@@ -75,7 +75,7 @@ final class QueriesLoaderCollector extends Collector {
     @Override
     public void setNextReader(AtomicReaderContext context) throws IOException {
         reader = context.reader();
-        idValues = idFieldData.load(context).getBytesValues();
+        idValues = idFieldData.load(context).getBytesValues(true);
     }
 
     @Override
diff --git a/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/src/main/java/org/elasticsearch/percolator/PercolatorService.java
index 3d22f75fe9a..0374157a21d 100644
--- a/src/main/java/org/elasticsearch/percolator/PercolatorService.java
+++ b/src/main/java/org/elasticsearch/percolator/PercolatorService.java
@@ -714,7 +714,7 @@ public class PercolatorService extends AbstractComponent {
                 for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
                     int segmentIdx = ReaderUtil.subIndex(scoreDoc.doc, percolatorSearcher.reader().leaves());
                     AtomicReaderContext atomicReaderContext = percolatorSearcher.reader().leaves().get(segmentIdx);
-                    BytesValues values = idFieldData.load(atomicReaderContext).getBytesValues();
+                    BytesValues values = idFieldData.load(atomicReaderContext).getBytesValues(true);
                     final int localDocId = scoreDoc.doc - atomicReaderContext.docBase;
                     assert values.hasValue(localDocId);
                     spare.bytes = values.getValue(localDocId);
diff --git a/src/main/java/org/elasticsearch/percolator/QueryCollector.java b/src/main/java/org/elasticsearch/percolator/QueryCollector.java
index dc0fafecc5b..28e1e1cd08d 100644
--- a/src/main/java/org/elasticsearch/percolator/QueryCollector.java
+++ b/src/main/java/org/elasticsearch/percolator/QueryCollector.java
@@ -103,7 +103,7 @@ abstract class QueryCollector extends Collector {
     @Override
     public void setNextReader(AtomicReaderContext context) throws IOException {
         // we use the UID because id might not be indexed
-        values = idFieldData.load(context).getBytesValues();
+        values = idFieldData.load(context).getBytesValues(true);
         if (facetCollector != null) {
             facetCollector.setNextReader(context);
         }
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetExecutor.java b/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetExecutor.java
index 3f417890925..799364ad21f 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetExecutor.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetExecutor.java
@@ -107,7 +107,7 @@ public class FieldsTermsStringFacetExecutor extends FacetExecutor {
         @Override
         public void setNextReader(AtomicReaderContext context) throws IOException {
             for (int i = 0; i < indexFieldDatas.length; i++) {
-                values[i] = indexFieldDatas[i].load(context).getBytesValues();
+                values[i] = indexFieldDatas[i].load(context).getBytesValues(true);
             }
             if (script != null) {
                 script.setNextReader(context);
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetExecutor.java b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetExecutor.java
index 6f7346c5004..e5b7d4dfcf3 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetExecutor.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetExecutor.java
@@ -107,7 +107,7 @@ public class TermsStringFacetExecutor extends FacetExecutor {
 
         @Override
         public void setNextReader(AtomicReaderContext context) throws IOException {
-            values = indexFieldData.load(context).getBytesValues();
+            values = indexFieldData.load(context).getBytesValues(true);
             if (script != null) {
                 script.setNextReader(context);
             }
@@ -130,7 +130,7 @@ public class TermsStringFacetExecutor extends FacetExecutor {
         for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
             int maxDoc = readerContext.reader().maxDoc();
             if (indexFieldData instanceof IndexFieldData.WithOrdinals) {
-                BytesValues.WithOrdinals values = ((IndexFieldData.WithOrdinals) indexFieldData).load(readerContext).getBytesValues();
+                BytesValues.WithOrdinals values = ((IndexFieldData.WithOrdinals) indexFieldData).load(readerContext).getBytesValues(false);
                 Ordinals.Docs ordinals = values.ordinals();
                 // 0 = docs with no value for field, so start from 1 instead
                 for (long ord = Ordinals.MIN_ORDINAL; ord < ordinals.getMaxOrd(); ord++) {
@@ -138,7 +138,7 @@ public class TermsStringFacetExecutor extends FacetExecutor {
                     aggregator.addValue(value, value.hashCode(), values);
                 }
             } else {
-                BytesValues values = indexFieldData.load(readerContext).getBytesValues();
+                BytesValues values = indexFieldData.load(readerContext).getBytesValues(true);
                 for (int docId = 0; docId < maxDoc; docId++) {
                     final int size = values.setDocument(docId);
                     for (int i = 0; i < size; i++) {
diff --git a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetExecutor.java b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetExecutor.java
index 52013d09119..685ab672cb8 100644
--- a/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetExecutor.java
+++ b/src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringOrdinalsFacetExecutor.java
@@ -203,7 +203,7 @@ public class TermsStringOrdinalsFacetExecutor extends FacetExecutor {
                     aggregators.add(current);
                 }
             }
-            values = indexFieldData.load(context).getBytesValues();
+            values = indexFieldData.load(context).getBytesValues(false);
             current = new ReaderAggregator(values, ordinalsCacheAbove, cacheRecycler);
             ordinals = values.ordinals();
         }
diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetExecutor.java b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetExecutor.java
index 5fef2f29ff2..904c962ba68 100644
--- a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetExecutor.java
+++ b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetExecutor.java
@@ -131,7 +131,7 @@ public class TermsStatsStringFacetExecutor extends FacetExecutor {
 
         @Override
         public void setNextReader(AtomicReaderContext context) throws IOException {
-            keyValues = keyIndexFieldData.load(context).getBytesValues();
+            keyValues = keyIndexFieldData.load(context).getBytesValues(true);
             if (script != null) {
                 script.setNextReader(context);
             } else {
diff --git a/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java b/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java
index 45ac6965bf7..89343302014 100644
--- a/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java
+++ b/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java
@@ -66,7 +66,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         IndexFieldData indexFieldData = getForField("value");
         AtomicReaderContext readerContext = refreshReader();
         AtomicFieldData fieldData = indexFieldData.load(readerContext);
-        BytesValues values = fieldData.getBytesValues();
+        BytesValues values = fieldData.getBytesValues(randomBoolean());
         for (int i = 0; i < fieldData.getNumDocs(); ++i) {
             assertThat(values.hasValue(i), equalTo(true));
         }
@@ -82,7 +82,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
 
-        BytesValues bytesValues = fieldData.getBytesValues();
+        BytesValues bytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(bytesValues.isMultiValued(), equalTo(false));
 
@@ -98,7 +98,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertValues(bytesValues, 1, one());
         assertValues(bytesValues, 2, three());
 
-        BytesValues hashedBytesValues = fieldData.getBytesValues();
+        BytesValues hashedBytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
         assertThat(hashedBytesValues.hasValue(1), equalTo(true));
@@ -185,7 +185,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertThat(fieldData.getNumDocs(), equalTo(3));
 
         BytesValues bytesValues = fieldData
-                .getBytesValues();
+                .getBytesValues(randomBoolean());
 
         assertThat(bytesValues.isMultiValued(), equalTo(false));
 
@@ -201,7 +201,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertValues(bytesValues, 1, Strings.EMPTY_ARRAY);
 
 
-        BytesValues hashedBytesValues = fieldData.getBytesValues();
+        BytesValues hashedBytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
         assertThat(hashedBytesValues.hasValue(1), equalTo(false));
@@ -227,7 +227,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
 
-        BytesValues bytesValues = fieldData.getBytesValues();
+        BytesValues bytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(bytesValues.isMultiValued(), equalTo(true));
 
@@ -241,7 +241,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
 
         assertValues(bytesValues, 0, two(), four());
 
-        BytesValues hashedBytesValues = fieldData.getBytesValues();
+        BytesValues hashedBytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
         assertThat(hashedBytesValues.hasValue(1), equalTo(true));
@@ -280,7 +280,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
 
-        BytesValues bytesValues = fieldData.getBytesValues();
+        BytesValues bytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(bytesValues.isMultiValued(), equalTo(true));
 
@@ -295,7 +295,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertValues(bytesValues, 0, two(), four());
         assertValues(bytesValues, 1, Strings.EMPTY_ARRAY);
 
-        BytesValues hashedBytesValues = fieldData.getBytesValues();
+        BytesValues hashedBytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(true));
         assertThat(hashedBytesValues.hasValue(1), equalTo(false));
@@ -320,7 +320,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
 
         assertThat(fieldData.getNumDocs(), equalTo(3));
 
-        BytesValues bytesValues = fieldData.getBytesValues();
+        BytesValues bytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(bytesValues.isMultiValued(), equalTo(false));
 
@@ -335,7 +335,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
         assertValues(bytesValues, 0, Strings.EMPTY_ARRAY);
         assertValues(bytesValues, 1, Strings.EMPTY_ARRAY);
         assertValues(bytesValues, 2, Strings.EMPTY_ARRAY);
-        BytesValues hashedBytesValues = fieldData.getBytesValues();
+        BytesValues hashedBytesValues = fieldData.getBytesValues(randomBoolean());
 
         assertThat(hashedBytesValues.hasValue(0), equalTo(false));
         assertThat(hashedBytesValues.hasValue(1), equalTo(false));
diff --git a/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java b/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java
index 7e2fdeb2559..158f6eb56e1 100644
--- a/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java
+++ b/src/test/java/org/elasticsearch/index/fielddata/DuelFieldDataTests.java
@@ -354,8 +354,8 @@ public class DuelFieldDataTests extends AbstractFieldDataTests {
         assertThat(leftData.getNumDocs(), equalTo(rightData.getNumDocs()));
 
         int numDocs = leftData.getNumDocs();
-        BytesValues leftBytesValues = random.nextBoolean() ? leftData.getBytesValues() : leftData.getHashedBytesValues();
-        BytesValues rightBytesValues = random.nextBoolean() ? rightData.getBytesValues() : rightData.getHashedBytesValues();
+        BytesValues leftBytesValues = leftData.getBytesValues(random.nextBoolean());
+        BytesValues rightBytesValues = rightData.getBytesValues(random.nextBoolean());
         BytesRef leftSpare = new BytesRef();
         BytesRef rightSpare = new BytesRef();
         for (int i = 0; i < numDocs; i++) {
diff --git a/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTest.java b/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTest.java
index ad9271ab1ac..a5a6d5c8c87 100644
--- a/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTest.java
+++ b/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTest.java
@@ -33,11 +33,6 @@ import java.util.Random;
 
 import static org.hamcrest.Matchers.equalTo;
 
-import java.util.Random;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
 public class FilterFieldDataTest extends AbstractFieldDataTests {
 
     @Override
@@ -77,7 +72,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
                         .put("filter.frequency.min_segment_size", 100).put("filter.frequency.min", 0.0d).put("filter.frequency.max", random.nextBoolean() ? 100 : 0.5d));
                 IndexFieldData fieldData = ifdService.getForField(new FieldMapper.Names("high_freq"), fieldDataType, false);
                 AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
-                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
+                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues(randomBoolean());
                 Docs ordinals = bytesValues.ordinals();
                 assertThat(2L, equalTo(ordinals.getNumOrds()));
                 assertThat(1000, equalTo(ordinals.getNumDocs()));
@@ -90,7 +85,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
                         .put("filter.frequency.min_segment_size", 100).put("filter.frequency.min",  random.nextBoolean() ? 101 : 101d/200.0d).put("filter.frequency.max", 201));
                 IndexFieldData fieldData = ifdService.getForField(new FieldMapper.Names("high_freq"), fieldDataType, false);
                 AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
-                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
+                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues(randomBoolean());
                 Docs ordinals = bytesValues.ordinals();
                 assertThat(1L, equalTo(ordinals.getNumOrds()));
                 assertThat(1000, equalTo(ordinals.getNumDocs()));
@@ -103,7 +98,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
                         .put("filter.frequency.min_segment_size", 101).put("filter.frequency.min", random.nextBoolean() ? 101 : 101d/200.0d));
                 IndexFieldData fieldData = ifdService.getForField(new FieldMapper.Names("med_freq"), fieldDataType, false);
                 AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
-                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
+                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues(randomBoolean());
                 Docs ordinals = bytesValues.ordinals();
                 assertThat(2L, equalTo(ordinals.getNumOrds()));
                 assertThat(1000, equalTo(ordinals.getNumDocs()));
@@ -117,7 +112,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
                         .put("filter.frequency.min_segment_size", 101).put("filter.frequency.min", random.nextBoolean() ? 101 : 101d/200.0d));
                 IndexFieldData fieldData = ifdService.getForField(new FieldMapper.Names("med_freq"), fieldDataType, false);
                 AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
-                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
+                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues(randomBoolean());
                 Docs ordinals = bytesValues.ordinals();
                 assertThat(2L, equalTo(ordinals.getNumOrds()));
                 assertThat(1000, equalTo(ordinals.getNumDocs()));
@@ -134,7 +129,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
                         .put("filter.frequency.max", random.nextBoolean() ? 99 : 99d/200.0d)); // 100
                 IndexFieldData fieldData = ifdService.getForField(new FieldMapper.Names("high_freq"), fieldDataType, false);
                 AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
-                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
+                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues(randomBoolean());
                 Docs ordinals = bytesValues.ordinals();
                 assertThat(1L, equalTo(ordinals.getNumOrds()));
                 assertThat(1000, equalTo(ordinals.getNumDocs()));
@@ -179,7 +174,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
                         .put("filter.regex.pattern", "\\d"));
                 IndexFieldData fieldData = ifdService.getForField(new FieldMapper.Names("high_freq"), fieldDataType, false);
                 AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
-                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
+                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues(randomBoolean());
                 Docs ordinals = bytesValues.ordinals();
                 assertThat(1L, equalTo(ordinals.getNumOrds()));
                 assertThat(1000, equalTo(ordinals.getNumDocs()));
@@ -191,7 +186,7 @@ public class FilterFieldDataTest extends AbstractFieldDataTests {
                         .put("filter.regex.pattern", "\\d{1,2}"));
                 IndexFieldData fieldData = ifdService.getForField(new FieldMapper.Names("high_freq"), fieldDataType, false);
                 AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> loadDirect = (WithOrdinals<Strings>) fieldData.loadDirect(context);
-                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues();
+                BytesValues.WithOrdinals bytesValues = loadDirect.getBytesValues(randomBoolean());
                 Docs ordinals = bytesValues.ordinals();
                 assertThat(2L, equalTo(ordinals.getNumOrds()));
                 assertThat(1000, equalTo(ordinals.getNumDocs()));