LUCENE-4426: ValueSource implementations for DocValues fields.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1394513 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -20,6 +20,9 @@ Changes in backwards compatibility policy
 
 New Features
 
+* LUCENE-4426: New ValueSource implementations (in lucene/queries) for
+  DocValues fields. (Adrien Grand)
+
 * LUCENE-4410: FilteredQuery now exposes a FilterStrategy that exposes
   how filters are applied during query execution. (Simon Willnauer) 
 

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DateDocValuesFieldSource.java

@@ -0,0 +1,122 @@
+package org.apache.lucene.queries.function.valuesource;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.LongDocValues;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A {@link ValueSource} for {@link DocValues} dates, backed by
+ * {@link org.apache.lucene.index.DocValues.Type#FIXED_INTS_64}
+ * or {@link org.apache.lucene.index.DocValues.Type#VAR_INTS}.
+ * <p>
+ * If the segment has no {@link DocValues}, the default
+ * {@link org.apache.lucene.index.DocValues.Source} of type
+ * {@link org.apache.lucene.index.DocValues.Type#FIXED_INTS_64} will be used.
+ *
+ * @lucene.experimental
+ */
+public class DateDocValuesFieldSource extends DocValuesFieldSource {
+
+  private class DVDateValues extends LongDocValues {
+
+    private final Bits liveDocs;
+    private final DocValues.Source source;
+
+    public DVDateValues(ValueSource vs, DocValues.Source source, Bits liveDocs) {
+      super(vs);
+      this.liveDocs = liveDocs;
+      this.source = source;
+    }
+
+    @Override
+    public boolean exists(int doc) {
+      return liveDocs == null || liveDocs.get(doc);
+    }
+
+    @Override
+    public long longVal(int doc) {
+      return source.getInt(doc);
+    }
+
+    @Override
+    public boolean bytesVal(int doc, BytesRef target) {
+      source.getBytes(doc, target);
+      return true;
+    }
+
+    @Override
+    public Date objectVal(int doc) {
+      return new Date(longVal(doc));
+    }
+
+    @Override
+    public String strVal(int doc) {
+      return dateToString(objectVal(doc));
+    }
+
+  }
+
+  /**
+   * @param fieldName the name of the {@link DocValues} field
+   * @param direct    whether or not to use a direct {@link org.apache.lucene.index.DocValues.Source}
+   */
+  public DateDocValuesFieldSource(String fieldName, boolean direct) {
+    super(fieldName, direct);
+  }
+
+  @Override
+  public FunctionValues getValues(@SuppressWarnings("rawtypes") Map context, AtomicReaderContext readerContext) throws IOException {
+    final DocValues.Source source = getSource(readerContext.reader(), DocValues.Type.FIXED_INTS_64);
+    final Bits liveDocs = readerContext.reader().getLiveDocs();
+    switch (source.getType()) {
+      case FIXED_INTS_64:
+      case VAR_INTS:
+        if (source.hasArray() && source.getArray() instanceof long[]) {
+          final long[] values = (long[]) source.getArray();
+          return new DVDateValues(this, source, liveDocs) {
+
+            @Override
+            public long longVal(int doc) {
+              return values[doc];
+            }
+
+          };
+        }
+        return new DVDateValues(this, source, liveDocs);
+      default:
+        throw new IllegalStateException(getClass().getSimpleName() + " only works with 64-bits integer types, not " + source.getType());
+    }
+  }
+
+  /** Return the string representation of the provided {@link Date}.
+   */
+  protected String dateToString(Date date) {
+    return date.toString();
+  }
+
+}

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DocValuesFieldSource.java

@@ -0,0 +1,90 @@
+package org.apache.lucene.queries.function.valuesource;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.queries.function.ValueSource;
+
+/**
+ * A {@link ValueSource} that is based on a field's {@link DocValues}.
+ * @lucene.experimental
+ */
+public abstract class DocValuesFieldSource extends ValueSource {
+
+  protected final String fieldName;
+  protected final boolean direct;
+
+  protected DocValuesFieldSource(String fieldName, boolean direct) {
+    this.fieldName = fieldName;
+    this.direct = direct;
+  }
+
+  protected final DocValues.Source getSource(AtomicReader reader, DocValues.Type defaultType) throws IOException {
+    final DocValues vals = reader.docValues(fieldName);
+    if (vals == null) {
+      switch (defaultType) {
+        case BYTES_FIXED_SORTED:
+        case BYTES_VAR_SORTED:
+          return DocValues.getDefaultSortedSource(defaultType, reader.maxDoc());
+        default:
+          return DocValues.getDefaultSource(defaultType);
+      }
+    }
+    return direct ? vals.getDirectSource() : vals.getSource();
+  }
+
+  /**
+   * @return whether or not a direct
+   * {@link org.apache.lucene.index.DocValues.Source} is used.
+   */
+  public boolean isDirect() {
+    return direct;
+  }
+
+  /**
+   * @return the field name
+   */
+  public String getFieldName() {
+    return fieldName;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o == null || !getClass().isInstance(o)) {
+      return false;
+    }
+    final DocValuesFieldSource other = (DocValuesFieldSource) o;
+    return fieldName.equals(other.fieldName) && direct == other.direct;
+  }
+
+  @Override
+  public int hashCode() {
+    int h = getClass().hashCode();
+    h = 31 * h + fieldName.hashCode();
+    h = 31 * h + (direct ? 1 : 0);
+    return h;
+  }
+
+  @Override
+  public String description() {
+    return fieldName;
+  }
+}

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumericDocValuesFieldSource.java

@@ -0,0 +1,285 @@
+package org.apache.lucene.queries.function.valuesource;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.queries.function.docvalues.FloatDocValues;
+import org.apache.lucene.queries.function.docvalues.IntDocValues;
+import org.apache.lucene.queries.function.docvalues.LongDocValues;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A {@link ValueSource} for numeric {@link DocValues} types:<ul>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#FLOAT_32},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#FLOAT_64},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#FIXED_INTS_8},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#FIXED_INTS_16},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#FIXED_INTS_32},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#FIXED_INTS_64},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#VAR_INTS}.</li></ul>
+ * <p>
+ * If the segment has no {@link DocValues}, the default
+ * {@link org.apache.lucene.index.DocValues.Source} of type
+ * {@link org.apache.lucene.index.DocValues.Type#FLOAT_64} will be used.
+ *
+ * @lucene.experimental
+ */
+public final class NumericDocValuesFieldSource extends DocValuesFieldSource {
+
+  private static abstract class DVIntValues extends IntDocValues {
+
+    private final Bits liveDocs;
+    private final DocValues.Source source;
+
+    public DVIntValues(ValueSource vs, DocValues.Source source, Bits liveDocs) {
+      super(vs);
+      this.liveDocs = liveDocs;
+      this.source = source;
+    }
+
+    @Override
+    public boolean exists(int doc) {
+      return liveDocs == null || liveDocs.get(doc);
+    }
+
+    @Override
+    public boolean bytesVal(int doc, BytesRef target) {
+      source.getBytes(doc, target);
+      return true;
+    }
+
+    @Override
+    public int intVal(int doc) {
+      return (int) source.getInt(doc);
+    }
+  }
+
+  private static class DVLongValues extends LongDocValues {
+
+    private final Bits liveDocs;
+    private final DocValues.Source source;
+
+    public DVLongValues(ValueSource vs, DocValues.Source source, Bits liveDocs) {
+      super(vs);
+      this.liveDocs = liveDocs;
+      this.source = source;
+    }
+
+    @Override
+    public boolean exists(int doc) {
+      return liveDocs == null || liveDocs.get(doc);
+    }
+
+    @Override
+    public boolean bytesVal(int doc, BytesRef target) {
+      source.getBytes(doc, target);
+      return true;
+    }
+
+    @Override
+    public long longVal(int doc) {
+      return source.getInt(doc);
+    }
+  }
+
+  private static abstract class DVDoubleValues extends DoubleDocValues {
+
+    private final Bits liveDocs;
+    private final DocValues.Source source;
+
+    public DVDoubleValues(ValueSource vs, DocValues.Source source, Bits liveDocs) {
+      super(vs);
+      this.liveDocs = liveDocs;
+      this.source = source;
+    }
+
+    @Override
+    public boolean exists(int doc) {
+      return liveDocs == null || liveDocs.get(doc);
+    }
+
+    @Override
+    public boolean bytesVal(int doc, BytesRef target) {
+      source.getBytes(doc, target);
+      return true;
+    }
+
+    @Override
+    public double doubleVal(int doc) {
+      return source.getFloat(doc);
+    }
+  }
+
+  /**
+   * @param fieldName the name of the {@link DocValues} field
+   * @param direct    whether or not to use a direct {@link org.apache.lucene.index.DocValues.Source}
+   */
+  public NumericDocValuesFieldSource(String fieldName, boolean direct) {
+    super(fieldName, direct);
+  }
+
+  @Override
+  public FunctionValues getValues(@SuppressWarnings("rawtypes") Map context, AtomicReaderContext readerContext) throws IOException {
+    final DocValues.Source source = getSource(readerContext.reader(), DocValues.Type.FLOAT_64);
+    final Bits liveDocs = readerContext.reader().getLiveDocs();
+    switch (source.getType()) {
+      case FIXED_INTS_8:
+      case FIXED_INTS_16:
+      case FIXED_INTS_32:
+      case FIXED_INTS_64:
+      case VAR_INTS:
+        if (source.hasArray()) {
+          final Object valuesArr = source.getArray();
+          if (valuesArr instanceof long[]) {
+            final long[] values = (long[]) source.getArray();
+            return new DVLongValues(this, source, liveDocs) {
+
+              @Override
+              public long longVal(int doc) {
+                return values[doc];
+              }
+
+            };
+          } else if (valuesArr instanceof int[]) {
+            final int[] values = (int[]) source.getArray();
+            return new DVIntValues(this, source, liveDocs) {
+
+              @Override
+              public int intVal(int doc) {
+                return values[doc];
+              }
+
+            };
+          } else if (valuesArr instanceof short[]) {
+            final short[] values = (short[]) source.getArray();
+            return new DVIntValues(this, source, liveDocs) {
+
+              @Override
+              public int intVal(int doc) {
+                return values[doc];
+              }
+
+              @Override
+              public Object objectVal(int doc) {
+                return shortVal(doc);
+              }
+
+            };
+          } else if (valuesArr instanceof byte[]) {
+            final byte[] values = (byte[]) source.getArray();
+            return new DVIntValues(this, source, liveDocs) {
+
+              @Override
+              public int intVal(int doc) {
+                return values[doc];
+              }
+
+              @Override
+              public Object objectVal(int doc) {
+                return byteVal(doc);
+              }
+
+            };
+          }
+        }
+        return new DVLongValues(this, source, liveDocs) {
+
+          @Override
+          public Object objectVal(int doc) {
+            switch (source.getType()) {
+              case FIXED_INTS_8:
+                return byteVal(doc);
+              case FIXED_INTS_16:
+                return shortVal(doc);
+              case FIXED_INTS_32:
+                return intVal(doc);
+              case FIXED_INTS_64:
+              case VAR_INTS:
+                return longVal(doc);
+              default:
+                throw new AssertionError();
+            }
+          }
+
+        };
+      case FLOAT_32:
+      case FLOAT_64:
+        if (source.hasArray()) {
+          final Object valuesArr = source.getArray();
+          if (valuesArr instanceof float[]) {
+            final float[] values = (float[]) valuesArr;
+            return new FloatDocValues(this) {
+
+              @Override
+              public boolean exists(int doc) {
+                return liveDocs == null || liveDocs.get(doc);
+              }
+
+              @Override
+              public boolean bytesVal(int doc, BytesRef target) {
+                source.getBytes(doc, target);
+                return true;
+              }
+
+              @Override
+              public float floatVal(int doc) {
+                return values[doc];
+              }
+
+            };
+          } else if (valuesArr instanceof double[]) {
+            final double[] values = (double[]) valuesArr;
+            return new DVDoubleValues(this, source, liveDocs) {
+
+              @Override
+              public double doubleVal(int doc) {
+                return values[doc];
+              }
+
+            };
+          }
+        }
+        return new DVDoubleValues(this, source, liveDocs) {
+
+          @Override
+          public Object objectVal(int doc) {
+            switch (source.getType()) {
+              case FLOAT_32:
+                return floatVal(doc);
+              case FLOAT_64:
+                return doubleVal(doc);
+              default:
+                throw new AssertionError();
+            }
+          }
+
+        };
+      default:
+        throw new IllegalStateException(getClass().getSimpleName() + " only works with numeric types, not " + source.getType());
+    }
+  }
+
+}

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NumericIndexDocValueSource.java

@@ -30,8 +30,7 @@ import org.apache.lucene.queries.function.ValueSource;
  * This {@link ValueSource} is compatible with all numerical
  * {@link FunctionValues}
  * 
- * @lucene.experimental
+ * @deprecated Use {@link NumericDocValuesFieldSource} instead.
- * 
  */
 public class NumericIndexDocValueSource extends ValueSource {
 
@@ -63,6 +62,10 @@ public class NumericIndexDocValueSource extends ValueSource {
         }
       };
 
+    case FIXED_INTS_8:
+    case FIXED_INTS_16:
+    case FIXED_INTS_32:
+    case FIXED_INTS_64:
     case VAR_INTS:
       return new FunctionValues() {
         @Override

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/StrDocValuesFieldSource.java

@@ -0,0 +1,135 @@
+package org.apache.lucene.queries.function.valuesource;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.StrDocValues;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.PackedInts;
+
+/**
+ * A {@link ValueSource} for binary {@link DocValues} that represent an UTF-8
+ * encoded String using:<ul>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#BYTES_FIXED_DEREF},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#BYTES_FIXED_STRAIGHT},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#BYTES_VAR_DEREF},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#BYTES_VAR_STRAIGHT},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#BYTES_FIXED_SORTED},</li>
+ * <li>{@link org.apache.lucene.index.DocValues.Type#BYTES_VAR_SORTED}.</li></ul>
+ * <p>
+ * If the segment has no {@link DocValues}, the default
+ * {@link org.apache.lucene.index.DocValues.Source} of type
+ * {@link org.apache.lucene.index.DocValues.Type#BYTES_VAR_SORTED} will be used.
+ *
+ * @lucene.experimental
+ */
+public class StrDocValuesFieldSource extends DocValuesFieldSource {
+
+  private static class DVStrValues extends StrDocValues {
+
+    private final Bits liveDocs;
+    private final DocValues.Source source;
+
+    public DVStrValues(ValueSource vs, DocValues.Source source, Bits liveDocs) {
+      super(vs);
+      this.liveDocs = liveDocs;
+      this.source = source;
+    }
+
+    @Override
+    public boolean exists(int doc) {
+      return liveDocs == null || liveDocs.get(doc);
+    }
+
+    @Override
+    public boolean bytesVal(int doc, BytesRef target) {
+      source.getBytes(doc, target);
+      return true;
+    }
+
+    @Override
+    public String strVal(int doc) {
+      BytesRef utf8Bytes = new BytesRef();
+      source.getBytes(doc, utf8Bytes);
+      return utf8Bytes.utf8ToString();
+    }
+  }
+
+  /**
+   * @param fieldName the name of the {@link DocValues} field
+   * @param direct    whether or not to use a direct {@link org.apache.lucene.index.DocValues.Source}
+   */
+  public StrDocValuesFieldSource(String fieldName, boolean direct) {
+    super(fieldName, direct);
+  }
+
+  @Override
+  public FunctionValues getValues(@SuppressWarnings("rawtypes") Map context, AtomicReaderContext readerContext) throws IOException {
+    final DocValues.Source source = getSource(readerContext.reader(), DocValues.Type.BYTES_VAR_SORTED);
+    final Bits liveDocs = readerContext.reader().getLiveDocs();
+    switch (source.getType()) {
+      case BYTES_FIXED_DEREF:
+      case BYTES_FIXED_STRAIGHT:
+      case BYTES_VAR_DEREF:
+      case BYTES_VAR_STRAIGHT:
+        return new DVStrValues(this, source, liveDocs);
+      case BYTES_FIXED_SORTED:
+      case BYTES_VAR_SORTED:
+        final DocValues.SortedSource sortedSource = source.asSortedSource();
+        if (sortedSource.hasPackedDocToOrd()) {
+          final PackedInts.Reader docToOrd = sortedSource.getDocToOrd();
+          return new DVStrValues(this, source, liveDocs) {
+
+            @Override
+            public int ordVal(int doc) {
+              return (int) docToOrd.get(doc);
+            }
+
+            @Override
+            public int numOrd() {
+              return sortedSource.getValueCount();
+            }
+
+          };
+        }
+        return new DVStrValues(this, source, liveDocs) {
+
+          @Override
+          public int ordVal(int doc) {
+            return sortedSource.ord(doc);
+          }
+
+          @Override
+          public int numOrd() {
+            return sortedSource.getValueCount();
+          }
+
+        };
+      default:
+        throw new IllegalStateException(getClass().getSimpleName() + " only works with binary types, not " + source.getType());
+    }
+  }
+
+}

lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java

@@ -0,0 +1,283 @@
+package org.apache.lucene.queries.function;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.ByteDocValuesField;
+import org.apache.lucene.document.DerefBytesDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleDocValuesField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatDocValuesField;
+import org.apache.lucene.document.IntDocValuesField;
+import org.apache.lucene.document.LongDocValuesField;
+import org.apache.lucene.document.PackedLongDocValuesField;
+import org.apache.lucene.document.ShortDocValuesField;
+import org.apache.lucene.document.SortedBytesDocValuesField;
+import org.apache.lucene.document.StraightBytesDocValuesField;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.queries.function.valuesource.DateDocValuesFieldSource;
+import org.apache.lucene.queries.function.valuesource.NumericDocValuesFieldSource;
+import org.apache.lucene.queries.function.valuesource.StrDocValuesFieldSource;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+
+public class TestDocValuesFieldSources extends LuceneTestCase {
+
+  public void test(DocValues.Type type) throws IOException {
+    Directory d = newDirectory();
+    IndexWriterConfig iwConfig = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    final int nDocs = atLeast(50);
+    final Field id = new IntDocValuesField("id", 0);
+    final Field f;
+    switch (type) {
+      case BYTES_FIXED_DEREF:
+        f = new DerefBytesDocValuesField("dv", new BytesRef(), true);
+        break;
+      case BYTES_FIXED_SORTED:
+        f = new SortedBytesDocValuesField("dv", new BytesRef(), true);
+        break;
+      case BYTES_FIXED_STRAIGHT:
+        f = new StraightBytesDocValuesField("dv", new BytesRef(), true);
+        break;
+      case BYTES_VAR_DEREF:
+        f = new DerefBytesDocValuesField("dv", new BytesRef(), false);
+        break;
+      case BYTES_VAR_SORTED:
+        f = new SortedBytesDocValuesField("dv", new BytesRef(), false);
+        break;
+      case BYTES_VAR_STRAIGHT:
+        f = new StraightBytesDocValuesField("dv", new BytesRef(), false);
+        break;
+      case FIXED_INTS_8:
+        f = new ByteDocValuesField("dv", (byte) 0);
+        break;
+      case FIXED_INTS_16:
+        f = new ShortDocValuesField("dv", (short) 0);
+        break;
+      case FIXED_INTS_32:
+        f = new IntDocValuesField("dv", 0);
+        break;
+      case FIXED_INTS_64:
+        f = new LongDocValuesField("dv", 0L);
+        break;
+      case VAR_INTS:
+        f = new PackedLongDocValuesField("dv", 0L);
+        break;
+      case FLOAT_32:
+        f = new FloatDocValuesField("dv", 0f);
+        break;
+      case FLOAT_64:
+        f = new DoubleDocValuesField("dv", 0d);
+        break;
+      default:
+        throw new AssertionError();
+    }
+    Document document = new Document();
+    document.add(id);
+    document.add(f);
+
+    final Object[] vals = new Object[nDocs];
+
+    RandomIndexWriter iw = new RandomIndexWriter(random(), d, iwConfig);
+    for (int i = 0; i < nDocs; ++i) {
+      id.setIntValue(i);
+      switch (type) {
+        case BYTES_FIXED_DEREF:
+        case BYTES_FIXED_SORTED:
+        case BYTES_FIXED_STRAIGHT:
+          vals[i] = _TestUtil.randomFixedByteLengthUnicodeString(random(), 10);
+          f.setBytesValue(new BytesRef((String) vals[i]));
+          break;
+        case BYTES_VAR_DEREF:
+        case BYTES_VAR_SORTED:
+        case BYTES_VAR_STRAIGHT:
+          vals[i] = _TestUtil.randomSimpleString(random(), 20);
+          f.setBytesValue(new BytesRef((String) vals[i]));
+          break;
+        case FIXED_INTS_8:
+          vals[i] = (byte) random().nextInt(256);
+          f.setByteValue((Byte) vals[i]);
+          break;
+        case FIXED_INTS_16:
+          vals[i] = (short) random().nextInt(1 << 16);
+          f.setShortValue((Short) vals[i]);
+          break;
+        case FIXED_INTS_32:
+          vals[i] = random().nextInt();
+          f.setIntValue((Integer) vals[i]);
+          break;
+        case FIXED_INTS_64:
+        case VAR_INTS:
+          final int bitsPerValue = RandomInts.randomIntBetween(random(), 1, 31); // keep it an int
+          vals[i] = (long) random().nextInt((int) PackedInts.maxValue(bitsPerValue));
+          f.setLongValue((Long) vals[i]);
+          break;
+        case FLOAT_32:
+          vals[i] = random().nextFloat();
+          f.setFloatValue((Float) vals[i]);
+          break;
+        case FLOAT_64:
+          vals[i] = random().nextDouble();
+          f.setDoubleValue((Double) vals[i]);
+          break;
+      }
+      iw.addDocument(document);
+      if (random().nextBoolean() && i % 10 == 9) {
+        iw.commit();
+      }
+    }
+    iw.close();
+
+    DirectoryReader rd = DirectoryReader.open(d);
+    for (AtomicReaderContext leave : rd.leaves()) {
+      final FunctionValues ids = new NumericDocValuesFieldSource("id", false).getValues(null, leave);
+      final ValueSource vs;
+      final boolean direct = random().nextBoolean();
+      switch (type) {
+        case BYTES_FIXED_DEREF:
+        case BYTES_FIXED_SORTED:
+        case BYTES_FIXED_STRAIGHT:
+        case BYTES_VAR_DEREF:
+        case BYTES_VAR_SORTED:
+        case BYTES_VAR_STRAIGHT:
+          vs = new StrDocValuesFieldSource("dv", direct);
+          break;
+        case FLOAT_32:
+        case FLOAT_64:
+        case FIXED_INTS_8:
+        case FIXED_INTS_16:
+        case FIXED_INTS_32:
+          vs = new NumericDocValuesFieldSource("dv", direct);
+          break;
+        case FIXED_INTS_64:
+        case VAR_INTS:
+          if (random().nextBoolean()) {
+            vs = new NumericDocValuesFieldSource("dv", direct);
+          } else {
+            vs = new DateDocValuesFieldSource("dv", direct);
+          }
+          break;
+        default:
+          throw new AssertionError();
+      }
+      final FunctionValues values = vs.getValues(null, leave);
+      BytesRef bytes = new BytesRef();
+      for (int i = 0; i < leave.reader().maxDoc(); ++i) {
+        assertTrue(values.exists(i));
+        if (vs instanceof StrDocValuesFieldSource) {
+          assertTrue(values.objectVal(i) instanceof String);
+        } else if (vs instanceof NumericDocValuesFieldSource) {
+          assertTrue(values.objectVal(i) instanceof Number);
+          switch (type) {
+            case FIXED_INTS_8:
+              assertTrue(values.objectVal(i) instanceof Byte);
+              assertTrue(values.bytesVal(i, bytes));
+              assertEquals(1, bytes.length);
+              break;
+            case FIXED_INTS_16:
+              assertTrue(values.objectVal(i) instanceof Short);
+              assertTrue(values.bytesVal(i, bytes));
+              assertEquals(2, bytes.length);
+              break;
+            case FIXED_INTS_32:
+              assertTrue(values.objectVal(i) instanceof Integer);
+              assertTrue(values.bytesVal(i, bytes));
+              assertEquals(4, bytes.length);
+              break;
+            case FIXED_INTS_64:
+            case VAR_INTS:
+              assertTrue(values.objectVal(i) instanceof Long);
+              assertTrue(values.bytesVal(i, bytes));
+              assertEquals(8, bytes.length);
+              break;
+            case FLOAT_32:
+              assertTrue(values.objectVal(i) instanceof Float);
+              assertTrue(values.bytesVal(i, bytes));
+              assertEquals(4, bytes.length);
+              break;
+            case FLOAT_64:
+              assertTrue(values.objectVal(i) instanceof Double);
+              assertTrue(values.bytesVal(i, bytes));
+              assertEquals(8, bytes.length);
+              break;
+            default:
+              throw new AssertionError();
+          }
+        } else if (vs instanceof DateDocValuesFieldSource) {
+          assertTrue(values.objectVal(i) instanceof Date);
+        } else {
+          throw new AssertionError();
+        }
+        
+        Object expected = vals[ids.intVal(i)];
+        switch (type) {
+          case BYTES_VAR_SORTED:
+          case BYTES_FIXED_SORTED:
+            values.ordVal(i); // no exception
+            assertTrue(values.numOrd() >= 1);
+          case BYTES_FIXED_DEREF:
+          case BYTES_FIXED_STRAIGHT:
+          case BYTES_VAR_DEREF:
+          case BYTES_VAR_STRAIGHT:
+            assertEquals(expected, values.objectVal(i));
+            assertEquals(expected, values.strVal(i));
+            assertEquals(expected, values.objectVal(i));
+            assertEquals(expected, values.strVal(i));
+            assertTrue(values.bytesVal(i, bytes));
+            assertEquals(new BytesRef((String) expected), bytes);
+            break;
+          case FLOAT_32:
+            assertEquals(((Number) expected).floatValue(), values.floatVal(i), 0.001);
+            break;
+          case FLOAT_64:
+            assertEquals(((Number) expected).doubleValue(), values.doubleVal(i), 0.001d);
+            break;
+          case FIXED_INTS_8:
+          case FIXED_INTS_16:
+          case FIXED_INTS_32:
+          case FIXED_INTS_64:
+          case VAR_INTS:
+            assertEquals(((Number) expected).longValue(), values.longVal(i));
+            break;
+        }
+      }
+    }
+    rd.close();
+    d.close();
+  }
+
+  public void test() throws IOException {
+    for (DocValues.Type type : DocValues.Type.values()) {
+      test(type);
+    }
+  }
+
+}