LUCENE-6064: throw exception instead of doing nothing, when sorting/grouping etc on misconfigured field

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1640464 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -206,6 +206,10 @@ API Changes
   instance based on a Directory implementation passed to the factory method.
   See MIGRATE.txt for more details.  (Uwe Schindler, Robert Muir)
 
+* LUCENE-6062: Throw exception instead of silently doing nothing if you try to 
+  sort/group/etc on a misconfigured field (e.g. no docvalues, no UninvertingReader, etc).  
+  (Robert Muir)
+
 Bug Fixes
 
 * LUCENE-5650: Enforce read-only access to any path outside the temporary

lucene/core/src/java/org/apache/lucene/index/DocValues.java

@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -199,12 +200,31 @@ public final class DocValues {
   // some helpers, for transition from fieldcache apis.
   // as opposed to the LeafReader apis (which must be strict for consistency), these are lenient
   
+  // helper method: to give a nice error when LeafReader.getXXXDocValues returns null.
+  private static void checkField(LeafReader in, String field, DocValuesType... expected) {
+    FieldInfo fi = in.getFieldInfos().fieldInfo(field);
+    if (fi != null) {
+      DocValuesType actual = fi.getDocValuesType();
+      throw new IllegalStateException("unexpected docvalues type " + actual + 
+                                        " for field '" + field + "' " +
+                                        (expected.length == 1 
+                                        ? "(expected=" + expected[0]
+                                        : "(expected one of " + Arrays.toString(expected)) + "). " +
+                                        "Use UninvertingReader or index with docvalues.");
+    }
+  }
+  
   /**
-   * Returns NumericDocValues for the reader, or {@link #emptyNumeric()} if it has none. 
+   * Returns NumericDocValues for the field, or {@link #emptyNumeric()} if it has none. 
+   * @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
+   * @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
+   * @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#NUMERIC}.
+   * @throws IOException if an I/O error occurs.
    */
-  public static NumericDocValues getNumeric(LeafReader in, String field) throws IOException {
+  public static NumericDocValues getNumeric(LeafReader reader, String field) throws IOException {
-    NumericDocValues dv = in.getNumericDocValues(field);
+    NumericDocValues dv = reader.getNumericDocValues(field);
     if (dv == null) {
+      checkField(reader, field, DocValuesType.NUMERIC);
       return emptyNumeric();
     } else {
       return dv;
@@ -212,13 +232,19 @@ public final class DocValues {
   }
   
   /**
-   * Returns BinaryDocValues for the reader, or {@link #emptyBinary} if it has none. 
+   * Returns BinaryDocValues for the field, or {@link #emptyBinary} if it has none. 
+   * @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
+   * @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
+   * @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#BINARY}
+   *                               or {@link DocValuesType#SORTED}.
+   * @throws IOException if an I/O error occurs.
    */
-  public static BinaryDocValues getBinary(LeafReader in, String field) throws IOException {
+  public static BinaryDocValues getBinary(LeafReader reader, String field) throws IOException {
-    BinaryDocValues dv = in.getBinaryDocValues(field);
+    BinaryDocValues dv = reader.getBinaryDocValues(field);
     if (dv == null) {
-      dv = in.getSortedDocValues(field);
+      dv = reader.getSortedDocValues(field);
       if (dv == null) {
+        checkField(reader, field, DocValuesType.BINARY, DocValuesType.SORTED);
         return emptyBinary();
       }
     }
@@ -226,11 +252,16 @@ public final class DocValues {
   }
   
   /**
-   * Returns SortedDocValues for the reader, or {@link #emptySorted} if it has none. 
+   * Returns SortedDocValues for the field, or {@link #emptySorted} if it has none. 
+   * @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
+   * @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
+   * @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#SORTED}.
+   * @throws IOException if an I/O error occurs.
    */
-  public static SortedDocValues getSorted(LeafReader in, String field) throws IOException {
+  public static SortedDocValues getSorted(LeafReader reader, String field) throws IOException {
-    SortedDocValues dv = in.getSortedDocValues(field);
+    SortedDocValues dv = reader.getSortedDocValues(field);
     if (dv == null) {
+      checkField(reader, field, DocValuesType.SORTED);
       return emptySorted();
     } else {
       return dv;
@@ -238,29 +269,41 @@ public final class DocValues {
   }
   
   /**
-   * Returns SortedNumericDocValues for the reader, or {@link #emptySortedNumeric} if it has none. 
+   * Returns SortedNumericDocValues for the field, or {@link #emptySortedNumeric} if it has none. 
+   * @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
+   * @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
+   * @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#SORTED_NUMERIC}
+   *                               or {@link DocValuesType#NUMERIC}.
+   * @throws IOException if an I/O error occurs.
    */
-  public static SortedNumericDocValues getSortedNumeric(LeafReader in, String field) throws IOException {
+  public static SortedNumericDocValues getSortedNumeric(LeafReader reader, String field) throws IOException {
-    SortedNumericDocValues dv = in.getSortedNumericDocValues(field);
+    SortedNumericDocValues dv = reader.getSortedNumericDocValues(field);
     if (dv == null) {
-      NumericDocValues single = in.getNumericDocValues(field);
+      NumericDocValues single = reader.getNumericDocValues(field);
       if (single == null) {
-        return emptySortedNumeric(in.maxDoc());
+        checkField(reader, field, DocValuesType.SORTED_NUMERIC, DocValuesType.NUMERIC);
+        return emptySortedNumeric(reader.maxDoc());
       }
-      Bits bits = in.getDocsWithField(field);
+      Bits bits = reader.getDocsWithField(field);
       return singleton(single, bits);
     }
     return dv;
   }
   
   /**
-   * Returns SortedSetDocValues for the reader, or {@link #emptySortedSet} if it has none. 
+   * Returns SortedSetDocValues for the field, or {@link #emptySortedSet} if it has none. 
+   * @return docvalues instance, or an empty instance if {@code field} does not exist in this reader.
+   * @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
+   * @throws IllegalStateException if {@code field} has docvalues, but the type is not {@link DocValuesType#SORTED_SET}
+   *                               or {@link DocValuesType#SORTED}.
+   * @throws IOException if an I/O error occurs.
    */
-  public static SortedSetDocValues getSortedSet(LeafReader in, String field) throws IOException {
+  public static SortedSetDocValues getSortedSet(LeafReader reader, String field) throws IOException {
-    SortedSetDocValues dv = in.getSortedSetDocValues(field);
+    SortedSetDocValues dv = reader.getSortedSetDocValues(field);
     if (dv == null) {
-      SortedDocValues sorted = in.getSortedDocValues(field);
+      SortedDocValues sorted = reader.getSortedDocValues(field);
       if (sorted == null) {
+        checkField(reader, field, DocValuesType.SORTED, DocValuesType.SORTED_SET);
         return emptySortedSet();
       }
       return singleton(sorted);
@@ -269,12 +312,21 @@ public final class DocValues {
   }
   
   /**
-   * Returns Bits for the reader, or {@link Bits} matching nothing if it has none. 
+   * Returns Bits for the field, or {@link Bits} matching nothing if it has none. 
+   * @return bits instance, or an empty instance if {@code field} does not exist in this reader.
+   * @throws IllegalStateException if {@code field} exists, but was not indexed with docvalues.
+   * @throws IOException if an I/O error occurs.
    */
-  public static Bits getDocsWithField(LeafReader in, String field) throws IOException {
+  public static Bits getDocsWithField(LeafReader reader, String field) throws IOException {
-    Bits dv = in.getDocsWithField(field);
+    Bits dv = reader.getDocsWithField(field);
     if (dv == null) {
-      return new Bits.MatchNoBits(in.maxDoc());
+      assert DocValuesType.values().length == 6; // we just don't want NONE
+      checkField(reader, field, DocValuesType.BINARY, 
+                            DocValuesType.NUMERIC, 
+                            DocValuesType.SORTED, 
+                            DocValuesType.SORTED_NUMERIC, 
+                            DocValuesType.SORTED_SET);
+      return new Bits.MatchNoBits(reader.maxDoc());
     } else {
       return dv;
     }

lucene/core/src/test/org/apache/lucene/TestSearch.java

@@ -128,6 +128,7 @@ public class TestSearch extends LuceneTestCase {
         Document d = new Document();
         d.add(newTextField("contents", docs[j], Field.Store.YES));
         d.add(new IntField("id", j, Field.Store.NO));
+        d.add(new NumericDocValuesField("id", j));
         writer.addDocument(d);
       }
       writer.close();

lucene/core/src/test/org/apache/lucene/index/TestDocValues.java

@@ -0,0 +1,287 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+/** Tests helper methods in DocValues */
+public class TestDocValues extends LuceneTestCase {
+  
+  /** 
+   * If the field doesn't exist, we return empty instances:
+   * it can easily happen that a segment just doesn't have any docs with the field.
+   */
+  public void testEmptyIndex() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    iw.addDocument(new Document());
+    DirectoryReader dr = DirectoryReader.open(iw, true);
+    LeafReader r = getOnlySegmentReader(dr);
+    
+    // ok
+    assertNotNull(DocValues.getBinary(r, "bogus"));
+    assertNotNull(DocValues.getNumeric(r, "bogus"));
+    assertNotNull(DocValues.getSorted(r, "bogus"));
+    assertNotNull(DocValues.getSortedSet(r, "bogus"));
+    assertNotNull(DocValues.getSortedNumeric(r, "bogus"));
+    assertNotNull(DocValues.getDocsWithField(r, "bogus"));
+    
+    dr.close();
+    iw.close();
+    dir.close();
+  }
+  
+  /** 
+   * field just doesnt have any docvalues at all: exception
+   */
+  public void testMisconfiguredField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    Document doc = new Document();
+    doc.add(new StringField("foo", "bar", Field.Store.NO));
+    iw.addDocument(doc);
+    DirectoryReader dr = DirectoryReader.open(iw, true);
+    LeafReader r = getOnlySegmentReader(dr);
+   
+    // errors
+    try {
+      DocValues.getBinary(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSorted(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedSet(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getDocsWithField(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    
+    dr.close();
+    iw.close();
+    dir.close();
+  }
+  
+  /** 
+   * field with numeric docvalues
+   */
+  public void testNumericField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 3));
+    iw.addDocument(doc);
+    DirectoryReader dr = DirectoryReader.open(iw, true);
+    LeafReader r = getOnlySegmentReader(dr);
+    
+    // ok
+    assertNotNull(DocValues.getNumeric(r, "foo"));
+    assertNotNull(DocValues.getSortedNumeric(r, "foo"));
+    assertNotNull(DocValues.getDocsWithField(r, "foo"));
+    
+    // errors
+    try {
+      DocValues.getBinary(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSorted(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedSet(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    
+    dr.close();
+    iw.close();
+    dir.close();
+  }
+  
+  /** 
+   * field with binary docvalues
+   */
+  public void testBinaryField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    Document doc = new Document();
+    doc.add(new BinaryDocValuesField("foo", new BytesRef("bar")));
+    iw.addDocument(doc);
+    DirectoryReader dr = DirectoryReader.open(iw, true);
+    LeafReader r = getOnlySegmentReader(dr);
+    
+    // ok
+    assertNotNull(DocValues.getBinary(r, "foo"));
+    assertNotNull(DocValues.getDocsWithField(r, "foo"));
+    
+    // errors
+    try {
+      DocValues.getNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSorted(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedSet(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    
+    dr.close();
+    iw.close();
+    dir.close();
+  }
+  
+  /** 
+   * field with sorted docvalues
+   */
+  public void testSortedField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    Document doc = new Document();
+    doc.add(new SortedDocValuesField("foo", new BytesRef("bar")));
+    iw.addDocument(doc);
+    DirectoryReader dr = DirectoryReader.open(iw, true);
+    LeafReader r = getOnlySegmentReader(dr);
+    
+    // ok
+    assertNotNull(DocValues.getBinary(r, "foo"));
+    assertNotNull(DocValues.getSorted(r, "foo"));
+    assertNotNull(DocValues.getSortedSet(r, "foo"));
+    assertNotNull(DocValues.getDocsWithField(r, "foo"));
+    
+    // errors
+    try {
+      DocValues.getNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    
+    dr.close();
+    iw.close();
+    dir.close();
+  }
+  
+  /** 
+   * field with sortedset docvalues
+   */
+  public void testSortedSetField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    Document doc = new Document();
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("bar")));
+    iw.addDocument(doc);
+    DirectoryReader dr = DirectoryReader.open(iw, true);
+    LeafReader r = getOnlySegmentReader(dr);
+    
+    // ok
+    assertNotNull(DocValues.getSortedSet(r, "foo"));
+    assertNotNull(DocValues.getDocsWithField(r, "foo"));
+    
+    // errors
+    try {
+      DocValues.getBinary(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSorted(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    
+    dr.close();
+    iw.close();
+    dir.close();
+  }
+  
+  /** 
+   * field with sortednumeric docvalues
+   */
+  public void testSortedNumericField() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    Document doc = new Document();
+    doc.add(new SortedNumericDocValuesField("foo", 3));
+    iw.addDocument(doc);
+    DirectoryReader dr = DirectoryReader.open(iw, true);
+    LeafReader r = getOnlySegmentReader(dr);
+    
+    // ok
+    assertNotNull(DocValues.getSortedNumeric(r, "foo"));
+    assertNotNull(DocValues.getDocsWithField(r, "foo"));
+    
+    // errors
+    try {
+      DocValues.getBinary(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getNumeric(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSorted(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    try {
+      DocValues.getSortedSet(r, "foo");
+      fail();
+    } catch (IllegalStateException expected) {}
+    
+    dr.close();
+    iw.close();
+    dir.close();
+  }
+}

lucene/core/src/test/org/apache/lucene/search/TestCustomSearcherSort.java

@@ -28,10 +28,12 @@ import java.util.TreeMap;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 
 /** Unit test for sorting code. */
@@ -58,7 +60,7 @@ public class TestCustomSearcherSort extends LuceneTestCase {
       Document doc = new Document();
       if ((i % 5) != 0) { // some documents must not have an entry in the first
                           // sort field
-        doc.add(newStringField("publicationDate_", random.getLuceneDate(), Field.Store.YES));
+        doc.add(new SortedDocValuesField("publicationDate_", new BytesRef(random.getLuceneDate())));
       }
       if ((i % 7) == 0) { // some documents to match the query (see below)
         doc.add(newTextField("content", "test", Field.Store.YES));

lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java

@@ -24,6 +24,7 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.DocsEnum;
@@ -35,6 +36,7 @@ import org.apache.lucene.search.FilteredQuery.FilterStrategy;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BitDocIdSet;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
@@ -65,21 +67,25 @@ public class TestFilteredQuery extends LuceneTestCase {
     Document doc = new Document();
     doc.add (newTextField("field", "one two three four five", Field.Store.YES));
     doc.add (newTextField("sorter", "b", Field.Store.YES));
+    doc.add (new SortedDocValuesField("sorter", new BytesRef("b")));
     writer.addDocument (doc);
 
     doc = new Document();
     doc.add (newTextField("field", "one two three four", Field.Store.YES));
     doc.add (newTextField("sorter", "d", Field.Store.YES));
+    doc.add (new SortedDocValuesField("sorter", new BytesRef("d")));
     writer.addDocument (doc);
 
     doc = new Document();
     doc.add (newTextField("field", "one two three y", Field.Store.YES));
     doc.add (newTextField("sorter", "a", Field.Store.YES));
+    doc.add (new SortedDocValuesField("sorter", new BytesRef("a")));
     writer.addDocument (doc);
 
     doc = new Document();
     doc.add (newTextField("field", "one two x", Field.Store.YES));
     doc.add (newTextField("sorter", "c", Field.Store.YES));
+    doc.add (new SortedDocValuesField("sorter", new BytesRef("c")));
     writer.addDocument (doc);
 
     // tests here require single segment (eg try seed

lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java

@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
@@ -48,6 +49,7 @@ public class TestIndexSearcher extends LuceneTestCase {
       Document doc = new Document();
       doc.add(newStringField("field", Integer.toString(i), Field.Store.NO));
       doc.add(newStringField("field2", Boolean.toString(i % 2 == 0), Field.Store.NO));
+      doc.add(new SortedDocValuesField("field2", new BytesRef(Boolean.toString(i % 2 == 0))));
       iw.addDocument(doc);
     }
     reader = iw.getReader();

lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java

@@ -21,15 +21,13 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.DoubleDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StoredField;
@@ -63,22 +61,12 @@ public class TestSearchAfter extends LuceneTestCase {
           new SortField("double", SortField.Type.DOUBLE, false),
           new SortField("bytes", SortField.Type.STRING, false),
           new SortField("bytesval", SortField.Type.STRING_VAL, false),
-          new SortField("intdocvalues", SortField.Type.INT, false),
-          new SortField("floatdocvalues", SortField.Type.FLOAT, false),
-          new SortField("sortedbytesdocvalues", SortField.Type.STRING, false),
-          new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, false),
-          new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, false),
           new SortField("int", SortField.Type.INT, true),
           new SortField("long", SortField.Type.LONG, true),
           new SortField("float", SortField.Type.FLOAT, true),
           new SortField("double", SortField.Type.DOUBLE, true),
           new SortField("bytes", SortField.Type.STRING, true),
           new SortField("bytesval", SortField.Type.STRING_VAL, true),
-          new SortField("intdocvalues", SortField.Type.INT, true),
-          new SortField("floatdocvalues", SortField.Type.FLOAT, true),
-          new SortField("sortedbytesdocvalues", SortField.Type.STRING, true),
-          new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, true),
-          new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, true),
           SortField.FIELD_SCORE,
           SortField.FIELD_DOC,
         }));
@@ -136,26 +124,19 @@ public class TestSearchAfter extends LuceneTestCase {
     dir = newDirectory();
     RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
     int numDocs = atLeast(200);
+    Random r = random();
     for (int i = 0; i < numDocs; i++) {
       List<Field> fields = new ArrayList<>();
       fields.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
       fields.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
-      fields.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
+      fields.add(new NumericDocValuesField("byte", (byte) r.nextInt()));
-      fields.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
+      fields.add(new NumericDocValuesField("short", (short) r.nextInt()));
-      fields.add(new IntField("int", random().nextInt(), Field.Store.NO));
+      fields.add(new NumericDocValuesField("int", r.nextInt()));
-      fields.add(new LongField("long", random().nextLong(), Field.Store.NO));
+      fields.add(new NumericDocValuesField("long", r.nextLong()));
-
+      fields.add(new FloatDocValuesField("float", r.nextFloat()));
-      fields.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
+      fields.add(new DoubleDocValuesField("double", r.nextDouble()));
-      fields.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
+      fields.add(new SortedDocValuesField("bytes", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
-      fields.add(newStringField("bytes", TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
+      fields.add(new BinaryDocValuesField("bytesval", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
-      fields.add(newStringField("bytesval", TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
-      fields.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
-
-      fields.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
-      fields.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
-      fields.add(new SortedDocValuesField("sortedbytesdocvalues", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
-      fields.add(new SortedDocValuesField("sortedbytesdocvaluesval", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
-      fields.add(new BinaryDocValuesField("straightbytesdocvalues", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
 
       Document document = new Document();
       document.add(new StoredField("id", ""+i));

lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java

@@ -219,9 +219,9 @@ public class TestShardSearching extends ShardSearchingTestBase {
                 //sort = new Sort(SortField.FIELD_DOC);
                 sort = null;
               } else if (what == 2) {
-                sort = new Sort(new SortField[] {new SortField("docid_int", SortField.Type.INT, random().nextBoolean())});
+                sort = new Sort(new SortField[] {new SortField("docid_intDV", SortField.Type.INT, random().nextBoolean())});
               } else {
-                sort = new Sort(new SortField[] {new SortField("title", SortField.Type.STRING, random().nextBoolean())});
+                sort = new Sort(new SortField[] {new SortField("titleDV", SortField.Type.STRING, random().nextBoolean())});
               }
             }
           } else {

lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java

@@ -19,8 +19,11 @@ package org.apache.lucene.search;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatDocValuesField;
 import org.apache.lucene.document.FloatField;
 import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.CompositeReaderContext;
 import org.apache.lucene.index.IndexReader;
@@ -29,6 +32,7 @@ import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 
@@ -103,9 +107,9 @@ public class TestTopDocsMerge extends LuceneTestCase {
 
       for(int docIDX=0;docIDX<numDocs;docIDX++) {
         final Document doc = new Document();
-        doc.add(newStringField("string", TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
+        doc.add(new SortedDocValuesField("string", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
         doc.add(newTextField("text", content[random().nextInt(content.length)], Field.Store.NO));
-        doc.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
+        doc.add(new FloatDocValuesField("float", random().nextFloat()));
         final int intValue;
         if (random().nextInt(100) == 17) {
           intValue = Integer.MIN_VALUE;
@@ -114,7 +118,7 @@ public class TestTopDocsMerge extends LuceneTestCase {
         } else {
           intValue = random().nextInt();
         }
-        doc.add(new IntField("int", intValue, Field.Store.NO));
+        doc.add(new NumericDocValuesField("int", intValue));
         if (VERBOSE) {
           System.out.println("  doc=" + doc);
         }

lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java

@@ -21,12 +21,9 @@ import java.util.Arrays;
 import java.util.Collections;
 
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.DoubleDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.expressions.js.JavascriptCompiler;
 import org.apache.lucene.index.IndexReader;
@@ -68,16 +65,10 @@ public class TestExpressionSorts extends LuceneTestCase {
       Document document = new Document();
       document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
       document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
-      document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
+      document.add(new NumericDocValuesField("int", random().nextInt()));
-      document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
+      document.add(new NumericDocValuesField("long", random().nextLong()));
-      document.add(new IntField("int", random().nextInt(), Field.Store.NO));
+      document.add(new FloatDocValuesField("float", random().nextFloat()));
-      document.add(new LongField("long", random().nextLong(), Field.Store.NO));
+      document.add(new DoubleDocValuesField("double", random().nextDouble()));
-
-      document.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
-      document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
-
-      document.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
-      document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
       iw.addDocument(document);
     }
     reader = iw.getReader();
@@ -119,8 +110,6 @@ public class TestExpressionSorts extends LuceneTestCase {
           new SortField("long", SortField.Type.LONG, reversed),
           new SortField("float", SortField.Type.FLOAT, reversed),
           new SortField("double", SortField.Type.DOUBLE, reversed),
-          new SortField("intdocvalues", SortField.Type.INT, reversed),
-          new SortField("floatdocvalues", SortField.Type.FLOAT, reversed),
           new SortField("score", SortField.Type.SCORE)
       };
       Collections.shuffle(Arrays.asList(fields), random());

lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java

@@ -27,7 +27,6 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.IntField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.facet.FacetField;
@@ -209,7 +208,7 @@ public class TestTaxonomyFacetSumValueSource extends FacetTestCase {
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
 
     Document doc = new Document();
-    doc.add(new IntField("num", 10, Field.Store.NO));
+    doc.add(new NumericDocValuesField("num", 10));
     doc.add(new FacetField("a", "foo1"));
     writer.addDocument(config.build(taxoWriter, doc));
 

lucene/grouping/src/test/org/apache/lucene/search/grouping/DistinctValuesCollectorTest.java

@@ -440,6 +440,7 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
 
       Document doc = new Document();
       doc.add(new StringField("id", String.format(Locale.ROOT, "%09d", i), Field.Store.YES));
+      doc.add(new SortedDocValuesField("id", new BytesRef(String.format(Locale.ROOT, "%09d", i))));
       if (groupValue != null) {
         addField(doc, groupField, groupValue);
       }

lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java

@@ -229,6 +229,7 @@ public class GroupingSearchTest extends LuceneTestCase {
         newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
     Document doc = new Document();
     doc.add(newField("group", "foo", StringField.TYPE_NOT_STORED));
+    doc.add(new SortedDocValuesField("group", new BytesRef("foo")));
     w.addDocument(doc);
 
     IndexSearcher indexSearcher = newSearcher(w.getReader());

lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java

@@ -1284,6 +1284,7 @@ public class TestBlockJoin extends LuceneTestCase {
     RandomIndexWriter w = new RandomIndexWriter(random(), d);
     Document parent = new Document();
     parent.add(new StoredField("parentID", "0"));
+    parent.add(new SortedDocValuesField("parentID", new BytesRef("0")));
     parent.add(newTextField("parentText", "text", Field.Store.NO));
     parent.add(newStringField("isParent", "yes", Field.Store.NO));
 
@@ -1304,6 +1305,7 @@ public class TestBlockJoin extends LuceneTestCase {
     parent.add(newTextField("parentText", "text", Field.Store.NO));
     parent.add(newStringField("isParent", "yes", Field.Store.NO));
     parent.add(new StoredField("parentID", "1"));
+    parent.add(new SortedDocValuesField("parentID", new BytesRef("1")));
 
     // parent last:
     docs.add(parent);
@@ -1348,6 +1350,7 @@ public class TestBlockJoin extends LuceneTestCase {
     RandomIndexWriter w = new RandomIndexWriter(random(), d);
     Document parent = new Document();
     parent.add(new StoredField("parentID", "0"));
+    parent.add(new SortedDocValuesField("parentID", new BytesRef("0")));
     parent.add(newTextField("parentText", "text", Field.Store.NO));
     parent.add(newStringField("isParent", "yes", Field.Store.NO));
 
@@ -1368,6 +1371,8 @@ public class TestBlockJoin extends LuceneTestCase {
     parent.add(newTextField("parentText", "text", Field.Store.NO));
     parent.add(newStringField("isParent", "yes", Field.Store.NO));
     parent.add(new StoredField("parentID", "1"));
+    parent.add(new SortedDocValuesField("parentID", new BytesRef("1")));
+    
 
     // parent last:
     docs.add(parent);

lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java

@@ -238,12 +238,14 @@ public class TestJoinUtil extends LuceneTestCase {
     doc.add(new TextField("description", "random text", Field.Store.NO));
     doc.add(new TextField("name", "name1", Field.Store.NO));
     doc.add(new TextField(idField, "7", Field.Store.NO));
+    doc.add(new SortedDocValuesField(idField, new BytesRef("7")));
     w.addDocument(doc);
 
     // 1
     doc = new Document();
     doc.add(new TextField("price", "10.0", Field.Store.NO));
     doc.add(new TextField(idField, "2", Field.Store.NO));
+    doc.add(new SortedDocValuesField(idField, new BytesRef("2")));
     doc.add(new TextField(toField, "7", Field.Store.NO));
     w.addDocument(doc);
 
@@ -251,6 +253,7 @@ public class TestJoinUtil extends LuceneTestCase {
     doc = new Document();
     doc.add(new TextField("price", "20.0", Field.Store.NO));
     doc.add(new TextField(idField, "3", Field.Store.NO));
+    doc.add(new SortedDocValuesField(idField, new BytesRef("3")));
     doc.add(new TextField(toField, "7", Field.Store.NO));
     w.addDocument(doc);
 
@@ -266,6 +269,7 @@ public class TestJoinUtil extends LuceneTestCase {
     doc = new Document();
     doc.add(new TextField("price", "10.0", Field.Store.NO));
     doc.add(new TextField(idField, "5", Field.Store.NO));
+    doc.add(new SortedDocValuesField(idField, new BytesRef("5")));
     doc.add(new TextField(toField, "0", Field.Store.NO));
     w.addDocument(doc);
 
@@ -273,6 +277,7 @@ public class TestJoinUtil extends LuceneTestCase {
     doc = new Document();
     doc.add(new TextField("price", "20.0", Field.Store.NO));
     doc.add(new TextField(idField, "6", Field.Store.NO));
+    doc.add(new SortedDocValuesField(idField, new BytesRef("6")));
     doc.add(new TextField(toField, "0", Field.Store.NO));
     w.addDocument(doc);
 

lucene/queries/src/test/org/apache/lucene/queries/function/TestBoostedQuery.java

@@ -4,7 +4,7 @@ package org.apache.lucene.queries.function;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.RandomIndexWriter;
@@ -18,6 +18,7 @@ import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -55,7 +56,7 @@ public class TestBoostedQuery extends LuceneTestCase {
     iwConfig.setMergePolicy(newLogMergePolicy());
     RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);
     Document document = new Document();
-    Field idField = new StringField("id", "", Field.Store.NO);
+    Field idField = new SortedDocValuesField("id", new BytesRef());
     document.add(idField);
     iw.addDocument(document);
     ir = iw.getReader();

lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java

@@ -24,6 +24,7 @@ import com.spatial4j.core.shape.Shape;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -122,7 +123,8 @@ public class SpatialExample extends LuceneTestCase {
 
   private Document newSampleDocument(int id, Shape... shapes) {
     Document doc = new Document();
-    doc.add(new IntField("id", id, Field.Store.YES));
+    doc.add(new StoredField("id", id));
+    doc.add(new NumericDocValuesField("id", id));
     //Potentially more than one shape in this field is supported by some
     // strategies; see the javadocs of the SpatialStrategy impl to see.
     for (Shape shape : shapes) {

lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java

@@ -663,8 +663,8 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
 
   private int runQuery(IndexSearcher s, Query q) throws Exception {
     s.search(q, 10);
-    int hitCount = s.search(q, null, 10, new Sort(new SortField("title", SortField.Type.STRING))).totalHits;
+    int hitCount = s.search(q, null, 10, new Sort(new SortField("titleDV", SortField.Type.STRING))).totalHits;
-    final Sort dvSort = new Sort(new SortField("title", SortField.Type.STRING));
+    final Sort dvSort = new Sort(new SortField("titleDV", SortField.Type.STRING));
     int hitCount2 = s.search(q, null, 10, dvSort).totalHits;
     assertEquals(hitCount, hitCount2);
     return hitCount;

lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java

@@ -38,6 +38,7 @@ import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
@@ -162,6 +163,7 @@ public class LineFileDocs implements Closeable {
     final Field body;
     final Field id;
     final Field idNum;
+    final Field idNumDV;
     final Field date;
 
     public DocState(boolean useDocValues) {
@@ -193,9 +195,12 @@ public class LineFileDocs implements Closeable {
 
       if (useDocValues) {
         titleDV = new SortedDocValuesField("titleDV", new BytesRef());
+        idNumDV = new NumericDocValuesField("docid_intDV", 0);
         doc.add(titleDV);
+        doc.add(idNumDV);
       } else {
         titleDV = null;
+        idNumDV = null;
       }
     }
   }
@@ -244,6 +249,9 @@ public class LineFileDocs implements Closeable {
     final int i = id.getAndIncrement();
     docState.id.setStringValue(Integer.toString(i));
     docState.idNum.setIntValue(i);
+    if (docState.idNumDV != null) {
+      docState.idNumDV.setLongValue(i);
+    }
     return docState.doc;
   }
 }

solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml

@@ -201,9 +201,10 @@ NOTE: Tests expect every field in this schema to be sortable.
     <fieldtype name="str_dv_last" class="solr.StrField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
     <fieldtype name="str_dv_first" class="solr.StrField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>
 
-    <fieldtype name="bin" class="solr.SortableBinaryField" stored="true" indexed="true" />
+    <!-- note: all 'binary' fields have docvalues, because this fieldtype doesnt support indexing -->
-    <fieldtype name="bin_last" class="solr.SortableBinaryField" stored="true" indexed="true" sortMissingLast="true"/>
+    <fieldtype name="bin" class="solr.SortableBinaryField" stored="true" indexed="true" docValues="true"/>
-    <fieldtype name="bin_first" class="solr.SortableBinaryField" stored="true" indexed="true" sortMissingFirst="true"/>
+    <fieldtype name="bin_last" class="solr.SortableBinaryField" stored="true" indexed="true" docValues="true" sortMissingLast="true"/>
+    <fieldtype name="bin_first" class="solr.SortableBinaryField" stored="true" indexed="true" docValues="true" sortMissingFirst="true"/>
     <fieldtype name="bin_dv" class="solr.SortableBinaryField" stored="true" indexed="false" docValues="true"/>
     <fieldtype name="bin_dv_last" class="solr.SortableBinaryField" stored="true" indexed="false" docValues="true" sortMissingLast="true"/>
     <fieldtype name="bin_dv_first" class="solr.SortableBinaryField" stored="true" indexed="false" docValues="true" sortMissingFirst="true"/>