LUCENE-3070: enable docvalues by default for every codec

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/docvalues@1099717 13f79535-47bb-0310-9956-ffa450edef68

lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java

@@ -20,13 +20,19 @@ package org.apache.lucene.index.codecs.appending;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
@@ -133,10 +139,22 @@ public class AppendingCodec extends Codec {
     StandardPostingsReader.files(dir, segmentInfo, codecIdAsString, files);
     BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     StandardCodec.getStandardExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java

@@ -30,7 +30,7 @@ import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
 import org.apache.lucene.util.ArrayUtil;
 
 

lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java

@@ -32,7 +32,7 @@ import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.PerDocConsumer;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.TermsConsumer;
-import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
 import org.apache.lucene.index.values.DocValues;
 import org.apache.lucene.store.Directory;
 

lucene/src/java/org/apache/lucene/index/codecs/Codec.java

@@ -52,13 +52,9 @@ public abstract class Codec {
    *  use; else, those files may be deleted. */
   public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
   
-  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+  public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
-    return null;
-  }
   
-  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+  public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
-    return null;
-  }
 
   /**
    * Gathers files associated with this segment

lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.index.codecs.docvalues;
+package org.apache.lucene.index.codecs;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -16,87 +16,50 @@ package org.apache.lucene.index.codecs.docvalues;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import java.io.IOException;
 import java.util.Comparator;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.FieldsConsumer;
-import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.values.Writer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 
-/**
+public class DefaultDocValuesConsumer extends PerDocConsumer {
- * A codec that adds DocValues support to a given codec transparently.
+  private final String segmentName;
- * @lucene.experimental
+  private final int codecId;
- */
+  private final Directory directory;
-public class DocValuesCodec extends Codec {
+  private final AtomicLong bytesUsed;
-  private final Codec other;
   private final Comparator<BytesRef> comparator;
 
-  public DocValuesCodec(Codec other, Comparator<BytesRef> comparator) {
+  public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator) {
-    this.name = other.name;
+    this.segmentName = state.segmentName;
-    this.other = other;
+    this.codecId = state.codecId;
+    this.bytesUsed = state.bytesUsed;
+    this.directory = state.directory;
     this.comparator = comparator;
   }
   
-  public DocValuesCodec(Codec other) {
-    this(other, null);
-  }
-
   @Override
-  public PerDocConsumer docsConsumer(final PerDocWriteState state)
-      throws IOException {
-    return new PerDocConsumer() {
   public void close() throws IOException {
   }
 
   @Override
-      public DocValuesConsumer addValuesField(FieldInfo field)
+  public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
-          throws IOException {
+    return Writer.create(field.getDocValues(),
-        final DocValuesConsumer consumer = Writer.create(field.getDocValues(),
+        docValuesId(segmentName, codecId, field.number),
-            docValuesId(state.segmentName, state.codecId, field.number),
         // TODO can we have a compound file per segment and codec for
         // docvalues?
-            state.directory, comparator, state.bytesUsed);
+        directory, comparator, bytesUsed);
-        return consumer;
-      }
-    };
   }
   
-  @Override
+  public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
-  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DocValuesProducerBase(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
-  }
-
-  @Override
-  public FieldsConsumer fieldsConsumer(SegmentWriteState state)
-      throws IOException {
-    return other.fieldsConsumer(state);
-  }
-
-  @Override
-  public FieldsProducer fieldsProducer(SegmentReadState state)
-      throws IOException {
-    return other.fieldsProducer(state);
-  }
-  
-  static String docValuesId(String segmentsName, int codecID, int fieldId) {
-    return segmentsName + "_" + codecID + "-" + fieldId;
-  }
-
-  @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, int codecId,
       Set<String> files) throws IOException {
     FieldInfos fieldInfos = segmentInfo.getFieldInfos();
     boolean indexed = false;
@@ -131,14 +94,13 @@ public class DocValuesCodec extends Codec {
 
       }
     }
-    if (indexed) {
-      other.files(dir, segmentInfo, codecId, files);
-    }
   }
   
-  @Override
+  static String docValuesId(String segmentsName, int codecID, int fieldId) {
-  public void getExtensions(Set<String> extensions) {
+    return segmentsName + "_" + codecID + "-" + fieldId;
-    other.getExtensions(extensions);
+  }
+
+  public static void getDocValuesExtensions(Set<String> extensions) {
     extensions.add(Writer.DATA_EXTENSION);
     extensions.add(Writer.INDEX_EXTENSION);
   }

lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.index.codecs.docvalues;
+package org.apache.lucene.index.codecs;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -37,12 +37,12 @@ import org.apache.lucene.store.Directory;
  * 
  * @lucene.experimental
  */
-public class DocValuesProducerBase extends PerDocValues {
+public class DefaultDocValuesProducer extends PerDocValues {
 
   protected final TreeMap<String, DocValues> docValues = new TreeMap<String, DocValues>();
 
   /**
-   * Creates a new {@link DocValuesProducerBase} instance and loads all
+   * Creates a new {@link DefaultDocValuesProducer} instance and loads all
    * {@link DocValues} instances for this segment and codec.
    * 
    * @param si
@@ -56,7 +56,7 @@ public class DocValuesProducerBase extends PerDocValues {
    * @throws IOException
    *           if an {@link IOException} occurs
    */
-  protected DocValuesProducerBase(SegmentInfo si, Directory dir,
+  public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
       FieldInfos fieldInfo, int codecId) throws IOException {
     load(fieldInfo, si.name, si.docCount, dir, codecId);
   }
@@ -77,7 +77,7 @@ public class DocValuesProducerBase extends PerDocValues {
       if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
         final String field = fieldInfo.name;
         // TODO can we have a compound file per segment and codec for docvalues?
-        final String id = DocValuesCodec.docValuesId(segment, codecId, fieldInfo.number);
+        final String id = DefaultDocValuesConsumer.docValuesId(segment, codecId, fieldInfo.number);
         docValues.put(field, loadDocValues(docCount, dir, id, fieldInfo
             .getDocValues()));
       }

lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.index.codecs.docvalues;
+package org.apache.lucene.index.codecs;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more

lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java

@@ -19,7 +19,6 @@ import java.io.Closeable;
 import java.io.IOException;
 
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
 import org.apache.lucene.index.values.DocValues;
 
 /**

lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesCodecProvider.java

@@ -1,34 +0,0 @@
-package org.apache.lucene.index.codecs.docvalues;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.CodecProvider;
-
-/**
- * Simple Codec provider that wrapps all registered codecs into a {@link DocValuesCodec}
- */
-public class DocValuesCodecProvider extends CodecProvider {
-
-  @Override
-  public synchronized void register(Codec codec) {
-    if (codec instanceof DocValuesCodec) {
-      super.register(codec);
-    } else {
-      super.register(new DocValuesCodec(codec));
-    }
-  }
-}

lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java

@@ -22,11 +22,14 @@ import java.io.IOException;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 
 /** Codec that reads the pre-flex-indexing postings
  *  format.  It does not provide a writer because newly
@@ -78,4 +81,14 @@ public class PreFlexCodec extends Codec {
     extensions.add(TERMS_EXTENSION);
     extensions.add(TERMS_INDEX_EXTENSION);
   }
+
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return null;
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return null;
+  }
 }

lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java

@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.pulsing;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
@@ -28,8 +29,13 @@ import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.standard.StandardPostingsWriter;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
 import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -38,6 +44,7 @@ import org.apache.lucene.index.codecs.TermsIndexReaderBase;
 import org.apache.lucene.index.codecs.TermsIndexWriterBase;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 
 /** This codec "inlines" the postings for terms that have
  *  low docFreq.  It wraps another codec, which is used for
@@ -155,10 +162,22 @@ public class PulsingCodec extends Codec {
     StandardPostingsReader.files(dir, segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     StandardCodec.getStandardExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java

@@ -20,14 +20,21 @@ package org.apache.lucene.index.codecs.simpletext;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 
 /** For debugging, curiosity, transparency only!!  Do not
  *  use this codec in production.
@@ -63,10 +70,23 @@ public class SimpleTextCodec extends Codec {
   @Override
   public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
     files.add(getPostingsFileName(segmentInfo.name, ""+id));
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     extensions.add(POSTINGS_EXTENSION);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  // TODO: would be great if these used a plain text impl
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java

@@ -20,12 +20,17 @@ package org.apache.lucene.index.codecs.standard;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.TermsIndexWriterBase;
@@ -34,7 +39,10 @@ import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
 import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
 import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.BlockTermsReader;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
+import org.apache.lucene.index.values.Writer;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 
 /** Default codec. 
  *  @lucene.experimental */
@@ -135,11 +143,13 @@ public class StandardCodec extends Codec {
     StandardPostingsReader.files(dir, segmentInfo, codecId, files);
     BlockTermsReader.files(dir, segmentInfo, codecId, files);
     VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     getStandardExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
   }
 
   public static void getStandardExtensions(Set<String> extensions) {
@@ -148,4 +158,14 @@ public class StandardCodec extends Codec {
     BlockTermsReader.getExtensions(extensions);
     VariableGapTermsIndexReader.getIndexExtensions(extensions);
   }
+
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+  }
 }

lucene/src/java/org/apache/lucene/index/values/PerDocFieldValues.java

@@ -20,7 +20,7 @@ import java.util.Comparator;
 
 import org.apache.lucene.document.DocValuesField;
 import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
 import org.apache.lucene.util.BytesRef;
 
 /**

lucene/src/java/org/apache/lucene/index/values/Writer.java

@@ -20,7 +20,7 @@ import java.io.IOException;
 import java.util.Comparator;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.lucene.index.codecs.docvalues.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;

lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java

@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.mockintblock;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
@@ -33,8 +34,13 @@ import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
 import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
 import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
 import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -186,11 +192,12 @@ public class MockFixedIntBlockCodec extends Codec {
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) {
+  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     final String codecIdAsString = "" + codecId;
     SepPostingsReaderImpl.files(segmentInfo, codecIdAsString, files);
     BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
@@ -198,5 +205,16 @@ public class MockFixedIntBlockCodec extends Codec {
     SepPostingsWriterImpl.getExtensions(extensions);
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java

@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.mockintblock;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
@@ -33,8 +34,13 @@ import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
 import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
 import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput;
 import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -211,11 +217,12 @@ public class MockVariableIntBlockCodec extends Codec {
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) {
+  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     final String codecIdAsString = "" + codecId;
     SepPostingsReaderImpl.files(segmentInfo, codecIdAsString, files);
     BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
@@ -223,5 +230,16 @@ public class MockVariableIntBlockCodec extends Codec {
     SepPostingsWriterImpl.getExtensions(extensions);
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+  }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
   }
 }

lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java

@@ -26,16 +26,22 @@ import java.util.Set;
 
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.BlockTermsReader;
 import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.TermStats;
@@ -338,7 +344,7 @@ public class MockRandomCodec extends Codec {
     BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
     VariableGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
-    
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
     // hackish!
     Iterator<String> it = files.iterator();
     while(it.hasNext()) {
@@ -356,7 +362,19 @@ public class MockRandomCodec extends Codec {
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
     VariableGapTermsIndexReader.getIndexExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
     extensions.add(SEED_EXT);
     //System.out.println("MockRandom.getExtensions return " + extensions);
   }
+  
+  // can we make this more evil?
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+  }
 }

lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java

@@ -20,14 +20,20 @@ package org.apache.lucene.index.codecs.mocksep;
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.PerDocWriteState;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.BlockTermsReader;
@@ -130,16 +136,18 @@ public class MockSepCodec extends Codec {
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) {
+  public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
     final String codecIdAsString = "" + codecId;
     SepPostingsReaderImpl.files(segmentInfo, codecIdAsString, files);
     BlockTermsReader.files(dir, segmentInfo, codecIdAsString, files);
     FixedGapTermsIndexReader.files(dir, segmentInfo, codecIdAsString, files);
+    DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
   }
 
   @Override
   public void getExtensions(Set<String> extensions) {
     getSepExtensions(extensions);
+    DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
   }
 
   public static void getSepExtensions(Set<String> extensions) {
@@ -147,4 +155,14 @@ public class MockSepCodec extends Codec {
     BlockTermsReader.getExtensions(extensions);
     FixedGapTermsIndexReader.getIndexExtensions(extensions);
   }
+  
+  @Override
+  public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+    return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+  }
+
+  @Override
+  public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+  }
 }

lucene/src/test/org/apache/lucene/TestExternalCodecs.java

@@ -489,6 +489,16 @@ public class TestExternalCodecs extends LuceneTestCase {
       }
     }
 
+    @Override
+    public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+      return null;
+    }
+
+    @Override
+    public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+      return null;
+    }
+
     @Override
     public void getExtensions(Set<String> extensions) {
     }

lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java

@@ -33,10 +33,15 @@ import org.apache.lucene.index.codecs.BlockTermsReader;
 import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.CoreCodecProvider;
+import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsReaderBase;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
 import org.apache.lucene.index.codecs.TermsIndexReaderBase;
@@ -196,11 +201,13 @@ public class TestDocTermOrds extends LuceneTestCase {
       StandardPostingsReader.files(dir, segmentInfo, ""+id, files);
       BlockTermsReader.files(dir, segmentInfo, ""+id, files);
       FixedGapTermsIndexReader.files(dir, segmentInfo, ""+id, files);
+      DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
     }
 
     @Override
     public void getExtensions(Set<String> extensions) {
       getStandardExtensions(extensions);
+      DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
     }
 
     public static void getStandardExtensions(Set<String> extensions) {
@@ -209,6 +216,16 @@ public class TestDocTermOrds extends LuceneTestCase {
       BlockTermsReader.getExtensions(extensions);
       FixedGapTermsIndexReader.getIndexExtensions(extensions);
     }
+    
+    @Override
+    public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+      return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+    }
+
+    @Override
+    public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+      return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+    }
   }
 
   public void testRandom() throws Exception {

lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java

@@ -41,7 +41,6 @@ import org.apache.lucene.index.MultiPerDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.codecs.docvalues.DocValuesCodecProvider;
 import org.apache.lucene.index.values.DocValues.MissingValue;
 import org.apache.lucene.index.values.DocValues.Source;
 import org.apache.lucene.queryParser.ParseException;
@@ -76,13 +75,10 @@ public class TestDocValuesIndexing extends LuceneTestCase {
    * - run RAT
    */
 
-  private CodecProvider provider;
-
   @Before
   public void setUp() throws Exception {
     super.setUp();
-    provider = new DocValuesCodecProvider();
+    assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex"));
-    provider.copyFrom(CodecProvider.getDefault());
   }
   
   /*
@@ -105,7 +101,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
 
     writer.close(true);
 
-    IndexReader reader = IndexReader.open(dir, null, true, 1, provider);
+    IndexReader reader = IndexReader.open(dir, null, true, 1);
     assertTrue(reader.isOptimized());
 
     IndexSearcher searcher = new IndexSearcher(reader);
@@ -244,7 +240,6 @@ public class TestDocValuesIndexing extends LuceneTestCase {
     LogMergePolicy policy = new LogDocMergePolicy();
     cfg.setMergePolicy(policy);
     policy.setUseCompoundFile(useCompoundFile);
-    cfg.setCodecProvider(provider);
     return cfg;
   }
 

lucene/src/test/org/apache/lucene/search/TestSort.java

@@ -37,7 +37,6 @@ import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.docvalues.DocValuesCodecProvider;
 import org.apache.lucene.index.values.Type;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
@@ -228,10 +227,8 @@ public class TestSort extends LuceneTestCase {
   public void setUp() throws Exception {
     super.setUp();
     
-    //nocommit - enable doc values by default for all tests
+    //nocommit - we need to be able to run this test with preflex, but without docvalues!!!!
-    DocValuesCodecProvider provider = new DocValuesCodecProvider();
+    assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex"));
-    provider.copyFrom(CodecProvider.getDefault());
-    CodecProvider.setDefault(provider);
     full = getFullIndex();
     searchX = getXIndex();
     searchY = getYIndex();