LUCENE-3480: refactoring of docvalues params in Codec.java

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1177851 13f79535-47bb-0310-9956-ffa450edef68

lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java

@@ -137,22 +137,22 @@ public class AppendingCodec extends Codec {
     StandardPostingsReader.files(dir, segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     StandardCodec.getStandardExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
   
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new DefaultDocValuesProducer(state);
   }
 }

lucene/src/java/org/apache/lucene/index/codecs/Codec.java

@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 import java.util.Set;
 
 import org.apache.lucene.index.PerDocWriteState;
@@ -26,7 +25,6 @@ import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
 
 /** @lucene.experimental */
 public abstract class Codec {
@@ -34,22 +32,9 @@ public abstract class Codec {
   /** Unique name that's used to retrieve this codec when
    *  reading the index */
   public final String name;
-  protected final boolean dvUseCompoundFile;
-  protected final Comparator<BytesRef> docValuesSortComparator;
   
   protected Codec(String name) {
-    this(name, true);
-  }
-  
-  protected Codec(String name, boolean docValuesUseCompoundFile) {
-    this(name, docValuesUseCompoundFile, BytesRef.getUTF8SortedAsUnicodeComparator());
-  }
-
-  protected Codec(String name, boolean docValuesUseCompoundFile,
-      Comparator<BytesRef> docValuesSortComparator) {
     this.name = name;
-    this.dvUseCompoundFile = docValuesUseCompoundFile;
-    this.docValuesSortComparator = docValuesSortComparator;
   }
 
   /** Writes a new segment */
@@ -87,38 +72,6 @@ public abstract class Codec {
 
   /** Records all file extensions this codec uses */
   public abstract void getExtensions(Set<String> extensions);
-
-  /**
-   * Returns <code>true</code> iff compound file should be used for
-   * IndexDocValues, otherwise <code>false</code>. The default is
-   * <code>true</code>.
-   * <p>
-   * NOTE: To change the default value you need to subclass a {@link Codec} with
-   * a distinct name since this value is final and should not be changed to
-   * prevent the risk of a index corruption. This setting is private to a
-   * {@link Codec}. If you intend to change this value on an existing
-   * {@link Codec} re-indexing is required.
-   * 
-   * @return <code>true</code> iff compound file should be used for
-   *         IndexDocValues, otherwise <code>false</code>.
-   */
-  public boolean getDocValuesUseCFS() {
-    return dvUseCompoundFile;
-  }
-
-  /**
-   * Returns the {@link BytesRef} comparator for sorted IndexDocValue variants.
-   * The default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}.
-   * <p>
-   * NOTE: To change the default value you need to subclass a {@link Codec} with
-   * a distinct name since this value is final and should not be changed to
-   * prevent the risk of a index corruption. This setting is private to a
-   * {@link Codec}. If you intend to change this value on an existing
-   * {@link Codec} re-indexing is required.
-   */
-  public Comparator<BytesRef> getDocValuesSortComparator() {
-    return docValuesSortComparator;
-  }
   
   @Override
   public String toString() {

lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java

@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 import java.util.Set;
 
 import org.apache.lucene.index.FieldInfo;
@@ -26,113 +25,51 @@ import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.codecs.DocValuesWriterBase;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Counter;
 
 /**
- * 
+ * Default PerDocConsumer implementation that uses compound file.
  * @lucene.experimental
  */
-public class DefaultDocValuesConsumer extends PerDocConsumer {
+public class DefaultDocValuesConsumer extends DocValuesWriterBase {
-  private final String segmentName;
-  private final int codecId;
   private final Directory directory;
-  private final Counter bytesUsed;
-  private final Comparator<BytesRef> comparator;
-  private boolean useCompoundFile;
-  private final IOContext context;
   
-  public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator, boolean useCompoundFile) throws IOException {
+  public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException {
-    this.segmentName = state.segmentName;
+    super(state);
-    this.codecId = state.codecId;
-    this.bytesUsed = state.bytesUsed;
-    this.context = state.context;
     //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files?
-    this.directory = useCompoundFile ? new CompoundFileDirectory(state.directory,
+    this.directory = new CompoundFileDirectory(state.directory,
-        IndexFileNames.segmentFileName(segmentName, codecId,
+        IndexFileNames.segmentFileName(state.segmentName, state.codecId,
-            IndexFileNames.COMPOUND_FILE_EXTENSION), context, true) : state.directory;
+            IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true);
-    this.comparator = comparator;
-    this.useCompoundFile = useCompoundFile;
   }
-
+  
-  public void close() throws IOException {
+  @Override
-    if (useCompoundFile) {
+  protected Directory getDirectory() {
-      this.directory.close();
+    return directory;
-    }
   }
 
   @Override
-  public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
+  public void close() throws IOException {
-    return Writer.create(field.getDocValues(),
+    this.directory.close();
-        docValuesId(segmentName, codecId, field.number),
-        directory, comparator, bytesUsed, context);
   }
-  
+
   @SuppressWarnings("fallthrough")
-  public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
+  public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
-      Set<String> files, boolean useCompoundFile) throws IOException {
     FieldInfos fieldInfos = segmentInfo.getFieldInfos();
     for (FieldInfo fieldInfo : fieldInfos) {
       if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
-        String filename = docValuesId(segmentInfo.name, codecId,
+        files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
-            fieldInfo.number);
+        files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
-        if (useCompoundFile) {
+        assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); 
-          files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+        assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); 
-          files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+        return;
-          assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); 
-          assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); 
-          return;
-        } else {
-          switch (fieldInfo.getDocValues()) {
-          case BYTES_FIXED_DEREF:
-          case BYTES_VAR_DEREF:
-          case BYTES_VAR_SORTED:
-          case BYTES_FIXED_SORTED:
-          case BYTES_VAR_STRAIGHT:
-            files.add(IndexFileNames.segmentFileName(filename, "",
-                Writer.INDEX_EXTENSION));
-            assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
-                Writer.INDEX_EXTENSION));
-            // until here all types use an index
-          case BYTES_FIXED_STRAIGHT:
-          case FLOAT_32:
-          case FLOAT_64:
-          case VAR_INTS:
-          case FIXED_INTS_16:
-          case FIXED_INTS_32:
-          case FIXED_INTS_64:
-          case FIXED_INTS_8:
-            files.add(IndexFileNames.segmentFileName(filename, "",
-                Writer.DATA_EXTENSION));
-            assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
-                Writer.DATA_EXTENSION));
-            break;
-        
-          default:
-            assert false;
-          }
-        }
       }
     }
   }
   
-
+  public static void getExtensions(Set<String> extensions) {
-  static String docValuesId(String segmentsName, int codecID, int fieldId) {
+    extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
-    return segmentsName + "_" + codecID + "-" + fieldId;
+    extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
   }
-  
-  public static void getDocValuesExtensions(Set<String> extensions, boolean useCompoundFile) {
-    if (useCompoundFile) {
-      extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
-      extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
-    } else {
-      extensions.add(Writer.DATA_EXTENSION);
-      extensions.add(Writer.INDEX_EXTENSION);
-    }
-  }
-
 }

lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java

@@ -16,184 +16,50 @@ package org.apache.lucene.index.codecs;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
+import java.util.Map;
 import java.util.TreeMap;
 
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.values.Bytes;
+import org.apache.lucene.index.codecs.DocValuesReaderBase;
 import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.Floats;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.ValueType;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 
 /**
- * Abstract base class for FieldsProducer implementations supporting
+ * Default PerDocValues implementation that uses compound file.
- * {@link IndexDocValues}.
- * 
  * @lucene.experimental
  */
-public class DefaultDocValuesProducer extends PerDocValues {
+public class DefaultDocValuesProducer extends DocValuesReaderBase {
-
   protected final TreeMap<String, IndexDocValues> docValues;
-  private final boolean useCompoundFile;
+  private final Directory cfs;
-  private final Closeable cfs;
-  private final Comparator<BytesRef> sortComparator;
 
   /**
-   * 
    * Creates a new {@link DefaultDocValuesProducer} instance and loads all
    * {@link IndexDocValues} instances for this segment and codec.
-   * 
-   * @param si
-   *          the segment info to load the {@link IndexDocValues} for.
-   * @param dir
-   *          the directory to load the {@link IndexDocValues} from.
-   * @param fieldInfo
-   *          the {@link FieldInfos}
-   * @param codecId
-   *          the codec ID
-   * @param useCompoundFile
-   *          if <code>true</code> this producer opens a compound file to read
-   *          IndexDocValues fields, otherwise each field defines its own set of
-   *          files.
-   * @param sortComparator
-   *          defines the sort order for sorted IndexDocValues variants
-   * @throws IOException
-   *           if an {@link IOException} occurs
    */
-  public DefaultDocValuesProducer(SegmentInfo si, Directory dir, 
+  public DefaultDocValuesProducer(SegmentReadState state) throws IOException {
-      FieldInfos fieldInfo, int codecId, boolean useCompoundFile, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
+    cfs = new CompoundFileDirectory(state.dir, 
-    this.useCompoundFile = useCompoundFile;
+        IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), 
-    this.sortComparator = sortComparator;
+        state.context, false);
-    final Directory directory;
+    docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context);
-    if (useCompoundFile) {
-      cfs = directory = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
-    } else {
-      cfs = null;
-      directory = dir;
-    }
-    docValues = load(fieldInfo, si.name, si.docCount, directory, codecId, context);
-  }
-
-  /**
-   * Returns a {@link IndexDocValues} instance for the given field name or
-   * <code>null</code> if this field has no {@link IndexDocValues}.
-   */
-  @Override
-  public IndexDocValues docValues(String field) throws IOException {
-    return docValues.get(field);
-  }
-
-  // Only opens files... doesn't actually load any values
-  protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
-      String segment, int docCount, Directory dir, int codecId, IOContext context)
-      throws IOException {
-    TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
-    boolean success = false;
-    try {
-
-      for (FieldInfo fieldInfo : fieldInfos) {
-        if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
-          final String field = fieldInfo.name;
-          // TODO can we have a compound file per segment and codec for
-          // docvalues?
-          final String id = DefaultDocValuesConsumer.docValuesId(segment,
-              codecId, fieldInfo.number);
-          values.put(field,
-              loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), sortComparator, context));
-        }
-      }
-      success = true;
-    } finally {
-      if (!success) {
-        // if we fail we must close all opened resources if there are any
-        closeInternal(values.values());
-      }
-    }
-    return values;
   }
   
-
+  @Override
-  /**
+  protected Map<String,IndexDocValues> docValues() {
-   * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
+    return docValues;
-   * Codecs that use different implementations for a certain {@link ValueType} can
-   * simply override this method and return their custom implementations.
-   * 
-   * @param docCount
-   *          number of documents in the segment
-   * @param dir
-   *          the {@link Directory} to load the {@link IndexDocValues} from
-   * @param id
-   *          the unique file ID within the segment
-   * @param type
-   *          the type to load
-   * @param sortComparator byte comparator used by sorted variants
-   * @return a {@link IndexDocValues} instance for the given type
-   * @throws IOException
-   *           if an {@link IOException} occurs
-   * @throws IllegalArgumentException
-   *           if the given {@link ValueType} is not supported
-   */
-  protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
-      ValueType type, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
-    switch (type) {
-    case FIXED_INTS_16:
-    case FIXED_INTS_32:
-    case FIXED_INTS_64:
-    case FIXED_INTS_8:
-    case VAR_INTS:
-      return Ints.getValues(dir, id, docCount, type, context);
-    case FLOAT_32:
-      return Floats.getValues(dir, id, docCount, context);
-    case FLOAT_64:
-      return Floats.getValues(dir, id, docCount, context);
-    case BYTES_FIXED_STRAIGHT:
-      return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, sortComparator, context);
-    case BYTES_FIXED_DEREF:
-      return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, sortComparator, context);
-    case BYTES_FIXED_SORTED:
-      return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, sortComparator, context);
-    case BYTES_VAR_STRAIGHT:
-      return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, sortComparator, context);
-    case BYTES_VAR_DEREF:
-      return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, sortComparator, context);
-    case BYTES_VAR_SORTED:
-      return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, sortComparator, context);
-    default:
-      throw new IllegalStateException("unrecognized index values mode " + type);
-    }
-  }
-
-  public void close() throws IOException {
-    closeInternal(docValues.values());
-  }
-
-  private void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
-    final Collection<? extends Closeable> toClose;
-    if (useCompoundFile) {
-      final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
-      list.add(cfs);
-      toClose = list; 
-    } else {
-      toClose = closeables;
-    } 
-    IOUtils.close(toClose);
   }
 
   @Override
-  public Collection<String> fields() {
+  protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
-    return docValues.keySet();
+    final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
+    list.add(cfs);
+    IOUtils.close(list);
   }
 }

lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java

@@ -0,0 +1,143 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.values.Bytes;
+import org.apache.lucene.index.values.Floats;
+import org.apache.lucene.index.values.IndexDocValues;
+import org.apache.lucene.index.values.Ints;
+import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Abstract base class for PerDocValues implementations
+ * @lucene.experimental
+ */
+public abstract class DocValuesReaderBase extends PerDocValues {
+  
+  protected abstract void closeInternal(Collection<? extends Closeable> closeables) throws IOException;
+  protected abstract Map<String, IndexDocValues> docValues();
+  
+  @Override
+  public void close() throws IOException {
+    closeInternal(docValues().values());
+  }
+  
+  @Override
+  public IndexDocValues docValues(String field) throws IOException {
+    return docValues().get(field);
+  }
+
+  @Override
+  public Collection<String> fields() {
+    return docValues().keySet();
+  }
+
+  public Comparator<BytesRef> getComparator() throws IOException {
+    return BytesRef.getUTF8SortedAsUnicodeComparator();
+  }
+  
+  // Only opens files... doesn't actually load any values
+  protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
+      String segment, int docCount, Directory dir, int codecId, IOContext context)
+      throws IOException {
+    TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
+    boolean success = false;
+    try {
+
+      for (FieldInfo fieldInfo : fieldInfos) {
+        if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
+          final String field = fieldInfo.name;
+          // TODO can we have a compound file per segment and codec for
+          // docvalues?
+          final String id = DefaultDocValuesConsumer.docValuesId(segment,
+              codecId, fieldInfo.number);
+          values.put(field,
+              loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context));
+        }
+      }
+      success = true;
+    } finally {
+      if (!success) {
+        // if we fail we must close all opened resources if there are any
+        closeInternal(values.values());
+      }
+    }
+    return values;
+  }
+  
+  /**
+   * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
+   * Codecs that use different implementations for a certain {@link ValueType} can
+   * simply override this method and return their custom implementations.
+   * 
+   * @param docCount
+   *          number of documents in the segment
+   * @param dir
+   *          the {@link Directory} to load the {@link IndexDocValues} from
+   * @param id
+   *          the unique file ID within the segment
+   * @param type
+   *          the type to load
+   * @return a {@link IndexDocValues} instance for the given type
+   * @throws IOException
+   *           if an {@link IOException} occurs
+   * @throws IllegalArgumentException
+   *           if the given {@link ValueType} is not supported
+   */
+  protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
+      ValueType type, IOContext context) throws IOException {
+    switch (type) {
+    case FIXED_INTS_16:
+    case FIXED_INTS_32:
+    case FIXED_INTS_64:
+    case FIXED_INTS_8:
+    case VAR_INTS:
+      return Ints.getValues(dir, id, docCount, type, context);
+    case FLOAT_32:
+      return Floats.getValues(dir, id, docCount, context);
+    case FLOAT_64:
+      return Floats.getValues(dir, id, docCount, context);
+    case BYTES_FIXED_STRAIGHT:
+      return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context);
+    case BYTES_FIXED_DEREF:
+      return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, getComparator(), context);
+    case BYTES_FIXED_SORTED:
+      return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, getComparator(), context);
+    case BYTES_VAR_STRAIGHT:
+      return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, getComparator(), context);
+    case BYTES_VAR_DEREF:
+      return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, getComparator(), context);
+    case BYTES_VAR_SORTED:
+      return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, getComparator(), context);
+    default:
+      throw new IllegalStateException("unrecognized index values mode " + type);
+    }
+  }
+}

lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java

@@ -0,0 +1,68 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.PerDocWriteState;
+import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Counter;
+
+/**
+ * Abstract base class for PerDocConsumer implementations
+ * @lucene.experimental
+ */
+public abstract class DocValuesWriterBase extends PerDocConsumer {
+  private final String segmentName;
+  private final int codecId;
+  private final Counter bytesUsed;
+  private final IOContext context;
+  
+  protected DocValuesWriterBase(PerDocWriteState state) {
+    this.segmentName = state.segmentName;
+    this.codecId = state.codecId;
+    this.bytesUsed = state.bytesUsed;
+    this.context = state.context;
+  }
+
+  protected abstract Directory getDirectory();
+  
+  @Override
+  public void close() throws IOException {   
+  }
+
+  @Override
+  public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
+    return Writer.create(field.getDocValues(),
+        docValuesId(segmentName, codecId, field.number),
+        getDirectory(), getComparator(), bytesUsed, context);
+  }
+
+  public static String docValuesId(String segmentsName, int codecID, int fieldId) {
+    return segmentsName + "_" + codecID + "-" + fieldId;
+  }
+  
+  public Comparator<BytesRef> getComparator() throws IOException {
+    return BytesRef.getUTF8SortedAsUnicodeComparator();
+  }
+}

lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java

@@ -51,5 +51,4 @@ public abstract class PerDocValues implements Closeable {
    * Returns all fields this {@link PerDocValues} contains values for.
    */
   public abstract Collection<String> fields();
-  
 }

lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java

@@ -787,22 +787,22 @@ public class MemoryCodec extends Codec {
   @Override
   public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION));
-    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     extensions.add(EXTENSION);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
 
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), IOContext.READONCE);
+    return new DefaultDocValuesProducer(state);
   }
 }

lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java

@@ -134,22 +134,22 @@ public class PulsingCodec extends Codec {
   public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
     StandardPostingsReader.files(dir, segmentInfo, codecID, files);
     BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     StandardCodec.getStandardExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
 
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new DefaultDocValuesProducer(state);
   }
 }

lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java

@@ -0,0 +1,92 @@
+package org.apache.lucene.index.codecs.sep;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PerDocWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.codecs.DocValuesWriterBase;
+import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.store.Directory;
+
+/**
+ * Implementation of PerDocConsumer that uses separate files.
+ * @lucene.experimental
+ */
+public class SepDocValuesConsumer extends DocValuesWriterBase {
+  private final Directory directory;
+  
+  public SepDocValuesConsumer(PerDocWriteState state) throws IOException {
+    super(state);
+    this.directory = state.directory;
+  }
+  
+  @Override
+  protected Directory getDirectory() {
+    return directory;
+  }
+
+  @SuppressWarnings("fallthrough")
+  public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
+      Set<String> files) throws IOException {
+    FieldInfos fieldInfos = segmentInfo.getFieldInfos();
+    for (FieldInfo fieldInfo : fieldInfos) {
+      if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
+        String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number);
+        switch (fieldInfo.getDocValues()) {
+          case BYTES_FIXED_DEREF:
+          case BYTES_VAR_DEREF:
+          case BYTES_VAR_SORTED:
+          case BYTES_FIXED_SORTED:
+          case BYTES_VAR_STRAIGHT:
+            files.add(IndexFileNames.segmentFileName(filename, "",
+                Writer.INDEX_EXTENSION));
+            assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+                Writer.INDEX_EXTENSION));
+            // until here all types use an index
+          case BYTES_FIXED_STRAIGHT:
+          case FLOAT_32:
+          case FLOAT_64:
+          case VAR_INTS:
+          case FIXED_INTS_16:
+          case FIXED_INTS_32:
+          case FIXED_INTS_64:
+          case FIXED_INTS_8:
+            files.add(IndexFileNames.segmentFileName(filename, "",
+                Writer.DATA_EXTENSION));
+            assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+                Writer.DATA_EXTENSION));
+            break;
+        
+          default:
+            assert false;
+        }
+      }
+    }
+  }
+
+  public static void getExtensions(Set<String> extensions) {
+    extensions.add(Writer.DATA_EXTENSION);
+    extensions.add(Writer.INDEX_EXTENSION);
+  }
+}

lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java

@@ -0,0 +1,54 @@
+package org.apache.lucene.index.codecs.sep;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.codecs.DocValuesReaderBase;
+import org.apache.lucene.index.values.IndexDocValues;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * Implementation of PerDocValues that uses separate files.
+ * @lucene.experimental
+ */
+public class SepDocValuesProducer extends DocValuesReaderBase {
+  private final TreeMap<String, IndexDocValues> docValues;
+
+  /**
+   * Creates a new {@link SepDocValuesProducer} instance and loads all
+   * {@link IndexDocValues} instances for this segment and codec.
+   */
+  public SepDocValuesProducer(SegmentReadState state) throws IOException {
+    docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.codecId, state.context);
+  }
+  
+  @Override
+  protected Map<String,IndexDocValues> docValues() {
+    return docValues;
+  }
+  
+  @Override
+  protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
+    IOUtils.close(closeables);
+  }
+}

lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java

@@ -69,23 +69,23 @@ public class SimpleTextCodec extends Codec {
   @Override
   public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     files.add(getPostingsFileName(segmentInfo.name, id));
-    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     extensions.add(POSTINGS_EXTENSION);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
   
   // TODO: would be great if these used a plain text impl
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new DefaultDocValuesProducer(state);
   }
 }

lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java

@@ -110,19 +110,19 @@ public class StandardCodec extends Codec {
   public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
     StandardPostingsReader.files(dir, segmentInfo, codecID, files);
     BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     getStandardExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
   }
 
   public static void getStandardExtensions(Set<String> extensions) {
     extensions.add(FREQ_EXTENSION);
     extensions.add(PROX_EXTENSION);
     BlockTreeTermsReader.getExtensions(extensions);
+    DefaultDocValuesConsumer.getExtensions(extensions);
   }
 
   @Override
@@ -132,11 +132,11 @@ public class StandardCodec extends Codec {
 
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new DefaultDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new DefaultDocValuesProducer(state);
   }
 }

lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java

@@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.sep.IntStreamFactory;
 import org.apache.lucene.index.codecs.sep.IntIndexInput;
 import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
 import org.apache.lucene.index.codecs.sep.SepPostingsReader;
 import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
 import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
 import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
@@ -207,7 +207,7 @@ public class MockFixedIntBlockCodec extends Codec {
     SepPostingsReader.files(segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+    SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
@@ -215,16 +215,16 @@ public class MockFixedIntBlockCodec extends Codec {
     SepPostingsWriter.getExtensions(extensions);
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    SepDocValuesConsumer.getExtensions(extensions);
   }
   
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new SepDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new SepDocValuesProducer(state);
   }
 }

lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java

@@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.sep.IntStreamFactory;
 import org.apache.lucene.index.codecs.sep.IntIndexInput;
 import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
 import org.apache.lucene.index.codecs.sep.SepPostingsReader;
 import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput;
 import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
 import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
@@ -230,7 +230,7 @@ public class MockVariableIntBlockCodec extends Codec {
     SepPostingsReader.files(segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+    SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
@@ -238,16 +238,16 @@ public class MockVariableIntBlockCodec extends Codec {
     SepPostingsWriter.getExtensions(extensions);
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    SepDocValuesConsumer.getExtensions(extensions);
   }
   
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new SepDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new SepDocValuesProducer(state);
   }
 }

lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java

@@ -58,6 +58,8 @@ import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter;
 import org.apache.lucene.index.codecs.sep.IntIndexInput;
 import org.apache.lucene.index.codecs.sep.IntIndexOutput;
 import org.apache.lucene.index.codecs.sep.IntStreamFactory;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
 import org.apache.lucene.index.codecs.sep.SepPostingsReader;
 import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
 import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
@@ -75,17 +77,13 @@ import org.apache.lucene.util._TestUtil;
  */
 
 public class MockRandomCodec extends Codec {
-
+  private final boolean useSepDocValues;
   private final Random seedRandom;
   private final String SEED_EXT = "sd";
   
   public MockRandomCodec(Random random) {
-    this(random, "MockRandom", true);
+    super("MockRandom");
-    
+    this.useSepDocValues = random.nextBoolean();
-  }
-  
-  protected MockRandomCodec(Random random, String name, boolean docValuesUseCompoundFile) {
-    super(name, docValuesUseCompoundFile);
     this.seedRandom = new Random(random.nextLong());
   }
 
@@ -428,7 +426,11 @@ public class MockRandomCodec extends Codec {
     BlockTreeTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
     VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+    if (useSepDocValues) {
+      SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
+    } else {
+      DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
+    }
     // hackish!
     Iterator<String> it = files.iterator();
     while(it.hasNext()) {
@@ -447,7 +449,11 @@ public class MockRandomCodec extends Codec {
     BlockTreeTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
     VariableGapTermsIndexReader.getIndexExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+    if (useSepDocValues) {
+      SepDocValuesConsumer.getExtensions(extensions);
+    } else {
+      DefaultDocValuesConsumer.getExtensions(extensions);      
+    }
     extensions.add(SEED_EXT);
     //System.out.println("MockRandom.getExtensions return " + extensions);
   }
@@ -455,11 +461,19 @@ public class MockRandomCodec extends Codec {
   // can we make this more evil?
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    if (useSepDocValues) {
+      return new SepDocValuesConsumer(state);
+    } else {
+      return new DefaultDocValuesConsumer(state);
+    }
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    if (useSepDocValues) {
+      return new SepDocValuesProducer(state);
+    } else {
+      return new DefaultDocValuesProducer(state);
+    }
   }
 }

lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomDocValuesCodec.java

@@ -1,15 +0,0 @@
-package org.apache.lucene.index.codecs.mockrandom;
-
-import java.util.Random;
-
-/**
- * Randomly combines terms index impl w/ postings impls. and uses non-CFS format for docvalues
- */
-public class MockRandomDocValuesCodec extends MockRandomCodec {
-
-  public MockRandomDocValuesCodec(Random random) {
-    super(random, "MockDocValuesCodec", false);
-    // uses noCFS for docvalues for test coverage
-  }
-
-}

lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java

@@ -25,13 +25,11 @@ import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
 import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
@@ -40,6 +38,8 @@ import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.TermsIndexReaderBase;
 import org.apache.lucene.index.codecs.TermsIndexWriterBase;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
 import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
 import org.apache.lucene.index.codecs.sep.SepPostingsReader;
 import org.apache.lucene.store.Directory;
@@ -139,28 +139,28 @@ public class MockSepCodec extends Codec {
     SepPostingsReader.files(segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
-    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+    SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     getSepExtensions(extensions);
-    DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
   }
 
   public static void getSepExtensions(Set<String> extensions) {
     SepPostingsWriter.getExtensions(extensions);
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
+    SepDocValuesConsumer.getExtensions(extensions);
   }
   
   @Override
   public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-    return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+    return new SepDocValuesConsumer(state);
   }
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+    return new SepDocValuesProducer(state);
   }
 }

lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java

@@ -43,7 +43,6 @@ import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec;
 import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec;
 import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
 import org.apache.lucene.index.codecs.mockrandom.MockRandomCodec;
-import org.apache.lucene.index.codecs.mockrandom.MockRandomDocValuesCodec;
 import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
 import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
 import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
@@ -281,8 +280,6 @@ public abstract class LuceneTestCase extends Assert {
     // baseBlockSize cannot be over 127:
     swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp);
     swapCodec(new MockRandomCodec(random), cp);
-    // give docvalues non-cfs testcoverage
-    swapCodec(new MockRandomDocValuesCodec(random), cp);
 
     return cp.lookup(codec);
   }

lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java

@@ -203,13 +203,13 @@ public class TestDocTermOrds extends LuceneTestCase {
       StandardPostingsReader.files(dir, segmentInfo, id, files);
       BlockTermsReader.files(dir, segmentInfo, id, files);
       FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
-      DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+      DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
     }
 
     @Override
     public void getExtensions(Set<String> extensions) {
       getStandardExtensions(extensions);
-      DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+      DefaultDocValuesConsumer.getExtensions(extensions);
     }
 
     public static void getStandardExtensions(Set<String> extensions) {
@@ -221,12 +221,12 @@ public class TestDocTermOrds extends LuceneTestCase {
     
     @Override
     public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
-      return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+      return new DefaultDocValuesConsumer(state);
     }
 
     @Override
     public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-      return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+      return new DefaultDocValuesProducer(state);
     }
   }