more 4.0 back compat

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1437608 13f79535-47bb-0310-9956-ffa450edef68

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java

@@ -87,19 +87,17 @@ public class Lucene40Codec extends Codec {
     return infosFormat;
   }
   
-  //nocommit need a read-only Lucene40DocValuesFormat / read-write in the impersonator
+  private final DocValuesFormat defaultDVFormat = new Lucene40LyingDocValuesFormat();
-  private final DocValuesFormat defaultDVFormat = new Lucene40DocValuesFormat();
 
   @Override
-  public final DocValuesFormat docValuesFormat() {
+  public DocValuesFormat docValuesFormat() {
     return defaultDVFormat;
   }
 
-  // nocommit need a read-only Lucene40NormsFormat / read-write in the impersonator
   private final NormsFormat normsFormat = new Lucene40NormsFormat();
 
   @Override
-  public final NormsFormat normsFormat() {
+  public NormsFormat normsFormat() {
     return normsFormat;
   }
 

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesFormat.java

@@ -20,52 +20,38 @@ package org.apache.lucene.codecs.lucene40;
 import java.io.IOException;
 
 import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
-
-// nocommit: still a lie, but allows javadocs @links to work
-// nocommit: make read-only and move to impersonator
-public class Lucene40DocValuesFormat extends Lucene42DocValuesFormat {
 
+// NOTE: not registered in SPI, doesnt respect segment suffix, etc
+// for back compat only!
+public class Lucene40DocValuesFormat extends DocValuesFormat {
+  
+  public Lucene40DocValuesFormat() {
+    super("Lucene40");
+  }
+  
   @Override
   public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-    final DocValuesConsumer delegate = super.fieldsConsumer(state);
+    throw new UnsupportedOperationException("this codec can only be used for reading");
-    return new DocValuesConsumer() {
-
-      @Override
-      public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
-        // hack: here we would examine the numerics and simulate in the impersonator the best we can
-        // e.g. if they are all in byte/int range write fixed, otherwise write packed or whatever
-        field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.VAR_INTS.name());
-        delegate.addNumericField(field, values);
-      }
-
-      @Override
-      public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
-        field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_STRAIGHT.name());
-        delegate.addBinaryField(field, values);
-      }
-
-      @Override
-      public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
-        field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_SORTED.name());
-        delegate.addSortedField(field, values, docToOrd);
-      }
-      
-      @Override
-      public void close() throws IOException {
-        delegate.close();
-      }
-    };
   }
-
+  
   @Override
   public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
-    return super.fieldsProducer(state);
+    String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, 
+                                                     "dv", 
+                                                     IndexFileNames.COMPOUND_FILE_EXTENSION);
+    return new Lucene40DocValuesReader(state, filename, Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY);
   }
+  
+  // constants for VAR_INTS
+  static final String VAR_INTS_CODEC_NAME = "PackedInts";
+  static final int VAR_INTS_VERSION_START = 0;
+  static final int VAR_INTS_VERSION_CURRENT = VAR_INTS_VERSION_START;
+  
+  static final byte VAR_INTS_PACKED = 0x00;
+  static final byte VAR_INTS_FIXED_64 = 0x01;
 }

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java

@@ -0,0 +1,137 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+class Lucene40DocValuesReader extends DocValuesProducer {
+  private final Directory dir;
+  private final SegmentReadState state;
+  private final String legacyKey;
+
+  // ram instances we have already loaded
+  private final Map<Integer,NumericDocValues> numericInstances = 
+      new HashMap<Integer,NumericDocValues>();
+  private final Map<Integer,BinaryDocValues> binaryInstances = 
+      new HashMap<Integer,BinaryDocValues>();
+  private final Map<Integer,SortedDocValues> sortedInstances = 
+      new HashMap<Integer,SortedDocValues>();
+  
+  Lucene40DocValuesReader(SegmentReadState state, String filename, String legacyKey) throws IOException {
+    this.state = state;
+    this.legacyKey = legacyKey;
+    this.dir = new CompoundFileDirectory(state.directory, filename, state.context, false);
+  }
+  
+  @Override
+  public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
+    NumericDocValues instance = numericInstances.get(field.number);
+    if (instance == null) {
+      switch(LegacyDocValuesType.valueOf(field.getAttribute(legacyKey))) {
+        case VAR_INTS:
+          instance = loadVarIntsField(field);
+          break;
+        default: 
+          throw new AssertionError(); // nocommit, implement the other types
+      }
+      numericInstances.put(field.number, instance);
+    }
+    return instance;
+  }
+  
+  private NumericDocValues loadVarIntsField(FieldInfo field) throws IOException {
+    String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, Integer.toString(field.number), "dat");
+    IndexInput input = dir.openInput(fileName, state.context);
+    boolean success = false;
+    try {
+      CodecUtil.checkHeader(input, Lucene40DocValuesFormat.VAR_INTS_CODEC_NAME, 
+                                   Lucene40DocValuesFormat.VAR_INTS_VERSION_START, 
+                                   Lucene40DocValuesFormat.VAR_INTS_VERSION_CURRENT);
+      byte header = input.readByte();
+      if (header == Lucene40DocValuesFormat.VAR_INTS_FIXED_64) {
+        int maxDoc = state.segmentInfo.getDocCount();
+        final long values[] = new long[maxDoc];
+        for (int i = 0; i < values.length; i++) {
+          values[i] = input.readLong();
+        }
+        return new NumericDocValues() {
+          @Override
+          public long get(int docID) {
+            return values[docID];
+          }
+        };
+      } else if (header == Lucene40DocValuesFormat.VAR_INTS_PACKED) {
+        final long minValue = input.readLong();
+        final long defaultValue = input.readLong();
+        final PackedInts.Reader reader = PackedInts.getReader(input);
+        return new NumericDocValues() {
+          @Override
+          public long get(int docID) {
+            final long value = reader.get(docID);
+            if (value == defaultValue) {
+              return 0;
+            } else {
+              return minValue + value;
+            }
+          }
+        };
+      } else {
+        throw new CorruptIndexException("invalid VAR_INTS header byte: " + header + " (resource=" + input + ")");
+      }
+    } finally {
+      if (success) {
+        IOUtils.close(input);
+      } else {
+        IOUtils.closeWhileHandlingException(input);
+      }
+    }
+  }
+
+  @Override
+  public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
+    throw new AssertionError();
+  }
+
+  @Override
+  public synchronized SortedDocValues getSorted(FieldInfo field) throws IOException {
+    throw new AssertionError();
+  }
+  
+  @Override
+  public void close() throws IOException {
+    dir.close();
+  }
+}

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingDocValuesFormat.java

@@ -0,0 +1,41 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+// nocommit: still a lie, but allows javadocs @links to work
+// nocommit: make read-only and move to impersonator
+public class Lucene40LyingDocValuesFormat extends Lucene42DocValuesFormat {
+
+  @Override
+  public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    throw new UnsupportedOperationException("this codec can only be used for reading");
+  }
+
+  @Override
+  public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
+    return super.fieldsProducer(state);
+  }
+}

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java

@@ -21,49 +21,23 @@ import java.io.IOException;
 
 import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
 
-// nocommit: still a lie, but allows javadocs @links to work
+public class Lucene40NormsFormat extends NormsFormat {
-// nocommit: make read-only and move to impersonator
+
-public class Lucene40NormsFormat extends Lucene42NormsFormat {
-  
   @Override
   public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
-    final DocValuesConsumer delegate = super.normsConsumer(state);
+    throw new UnsupportedOperationException("this codec can only be used for reading");
-    return new DocValuesConsumer() {
-
-      @Override
-      public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
-        // hack: here we would examine the numerics and simulate in the impersonator the best we can
-        // e.g. if they are all in byte/int range write fixed, otherwise write packed or whatever
-        field.putAttribute(Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY, LegacyDocValuesType.VAR_INTS.name());
-        delegate.addNumericField(field, values);
-      }
-      
-      @Override
-      public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
-        assert false;
-      }
-
-      @Override
-      public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
-        assert false;
-      }
-
-      @Override
-      public void close() throws IOException {
-        delegate.close();
-      }
-    };
   }
 
   @Override
   public DocValuesProducer normsProducer(SegmentReadState state) throws IOException {
-    return super.normsProducer(state);
+    String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, 
+                                                     "nrm", 
+                                                     IndexFileNames.COMPOUND_FILE_EXTENSION);
+    return new Lucene40DocValuesReader(state, filename, Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY);
   }
 }

lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java

@@ -32,13 +32,12 @@ import org.apache.lucene.codecs.StoredFieldsWriter;
 import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat;
 import org.apache.lucene.codecs.compressing.CompressionMode;
-import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40LyingDocValuesFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.Directory;
@@ -121,18 +120,16 @@ public class Lucene41Codec extends Codec {
   }
   
   @Override
-  public final DocValuesFormat docValuesFormat() {
+  public DocValuesFormat docValuesFormat() {
     return dvFormat;
   }
 
   private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
-  // nocommit need a read-only Lucene40DocValuesFormat / read-write in the impersonator
+  private final DocValuesFormat dvFormat = new Lucene40LyingDocValuesFormat();
-  private final DocValuesFormat dvFormat = new Lucene40DocValuesFormat();
-  // nocommit need a read-only Lucene40NormsFormat / read-write in the impersonator
   private final NormsFormat normsFormat = new Lucene40NormsFormat();
 
   @Override
-  public final NormsFormat normsFormat() {
+  public NormsFormat normsFormat() {
     return normsFormat;
   }
 }

lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java

@@ -0,0 +1,119 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+class Lucene40DocValuesWriter extends DocValuesConsumer {
+  private final Directory dir;
+  private final SegmentWriteState state;
+  private final String legacyKey;
+
+  // note: intentionally ignores seg suffix
+  // String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, "dv", IndexFileNames.COMPOUND_FILE_EXTENSION);
+  Lucene40DocValuesWriter(SegmentWriteState state, String filename, String legacyKey) throws IOException {
+    this.state = state;
+    this.legacyKey = legacyKey;
+    this.dir = new CompoundFileDirectory(state.directory, filename, state.context, true);
+  }
+  
+  @Override
+  public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+    // TODO: examine the values: and simulate all the possibilities.
+    // e.g. if all values fit in a byte, write a fixed_8 etc.
+    field.putAttribute(legacyKey, LegacyDocValuesType.VAR_INTS.name());
+    String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, Integer.toString(field.number), "dat");
+    IndexOutput data = dir.createOutput(fileName, state.context);
+    boolean success = false;
+    try {
+      addVarIntsField(data, values);
+      success = true;
+    } finally {
+      if (success) {
+        IOUtils.close(data);
+      } else {
+        IOUtils.closeWhileHandlingException(data);
+      }
+    }
+  }
+
+  @Override
+  public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+    assert false;
+  }
+
+  @Override
+  public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+    assert false;
+  }
+  
+  @Override
+  public void close() throws IOException {
+    dir.close();
+  }
+  
+  private void addVarIntsField(IndexOutput output, Iterable<Number> values) throws IOException {
+    long minValue = Long.MAX_VALUE;
+    long maxValue = Long.MIN_VALUE;
+    for (Number n : values) {
+      long v = n.longValue();
+      minValue = Math.min(minValue, v);
+      maxValue = Math.max(maxValue, v);
+    }
+    
+    CodecUtil.writeHeader(output, 
+                          Lucene40DocValuesFormat.VAR_INTS_CODEC_NAME, 
+                          Lucene40DocValuesFormat.VAR_INTS_VERSION_CURRENT);
+    
+    final long delta = maxValue - minValue;
+    
+    if (delta < 0) {
+      // writes longs
+      output.writeByte(Lucene40DocValuesFormat.VAR_INTS_FIXED_64);
+      for (Number n : values) {
+        output.writeLong(n.longValue());
+      }
+    } else {
+      // writes packed ints
+      output.writeByte(Lucene40DocValuesFormat.VAR_INTS_PACKED);
+      output.writeLong(minValue);
+      output.writeLong(0 - minValue); // default value (representation of 0)
+      PackedInts.Writer writer = PackedInts.getWriter(output, 
+                                                      state.segmentInfo.getDocCount(),
+                                                      PackedInts.bitsRequired(delta), 
+                                                      PackedInts.DEFAULT);
+      for (Number n : values) {
+        writer.add(n.longValue() - minValue);
+      }
+      writer.finish();
+    }
+  }
+}

lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40LyingRWDocValuesFormat.java

@@ -0,0 +1,65 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.BytesRef;
+
+public class Lucene40LyingRWDocValuesFormat extends Lucene40LyingDocValuesFormat {
+  private final DocValuesFormat lie = new Lucene42DocValuesFormat();
+
+  // nocommit: a lie
+  @Override
+  public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    final DocValuesConsumer delegate = lie.fieldsConsumer(state);
+    return new DocValuesConsumer() {
+
+      @Override
+      public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+        // hack: here we would examine the numerics and simulate in the impersonator the best we can
+        // e.g. if they are all in byte/int range write fixed, otherwise write packed or whatever
+        field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.VAR_INTS.name());
+        delegate.addNumericField(field, values);
+      }
+
+      @Override
+      public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+        field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_STRAIGHT.name());
+        delegate.addBinaryField(field, values);
+      }
+
+      @Override
+      public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+        field.putAttribute(Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY, LegacyDocValuesType.BYTES_VAR_SORTED.name());
+        delegate.addSortedField(field, values, docToOrd);
+      }
+      
+      @Override
+      public void close() throws IOException {
+        delegate.close();
+      }
+    };
+  }
+}

lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java

@@ -2,8 +2,10 @@ package org.apache.lucene.codecs.lucene40;
 
 import java.io.IOException;
 
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FieldInfosWriter;
+import org.apache.lucene.codecs.NormsFormat;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -31,8 +33,21 @@ public final class Lucene40RWCodec extends Lucene40Codec {
     }
   };
   
+  private final DocValuesFormat docValues = new Lucene40LyingRWDocValuesFormat();
+  private final NormsFormat norms = new Lucene40RWNormsFormat();
+  
   @Override
   public FieldInfosFormat fieldInfosFormat() {
     return fieldInfos;
   }
+
+  @Override
+  public DocValuesFormat docValuesFormat() {
+    return docValues;
+  }
+
+  @Override
+  public NormsFormat normsFormat() {
+    return norms;
+  }
 }

lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWDocValuesFormat.java

@@ -0,0 +1,35 @@
+package org.apache.lucene.codecs.lucene40;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+
+public class Lucene40RWDocValuesFormat extends Lucene40DocValuesFormat {
+
+  @Override
+  public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, 
+                                                     "dv", 
+                                                     IndexFileNames.COMPOUND_FILE_EXTENSION);
+    return new Lucene40DocValuesWriter(state, filename, Lucene40FieldInfosReader.LEGACY_DV_TYPE_KEY);
+  }
+}

lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWNormsFormat.java

@@ -0,0 +1,35 @@
+package org.apache.lucene.codecs.lucene40;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class Lucene40RWNormsFormat extends Lucene40NormsFormat {
+
+  @Override
+  public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
+    String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, 
+                                                     "nrm", 
+                                                     IndexFileNames.COMPOUND_FILE_EXTENSION);
+    return new Lucene40DocValuesWriter(state, filename, Lucene40FieldInfosReader.LEGACY_NORM_TYPE_KEY);
+  }
+}

lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java

@@ -2,11 +2,15 @@ package org.apache.lucene.codecs.lucene41;
 
 import java.io.IOException;
 
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FieldInfosWriter;
+import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosWriter;
+import org.apache.lucene.codecs.lucene40.Lucene40LyingRWDocValuesFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40RWNormsFormat;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -37,6 +41,9 @@ public class Lucene41RWCodec extends Lucene41Codec {
     }
   };
   
+  private final DocValuesFormat docValues = new Lucene40LyingRWDocValuesFormat();
+  private final NormsFormat norms = new Lucene40RWNormsFormat();
+  
   @Override
   public FieldInfosFormat fieldInfosFormat() {
     return fieldInfos;
@@ -46,4 +53,14 @@ public class Lucene41RWCodec extends Lucene41Codec {
   public StoredFieldsFormat storedFieldsFormat() {
     return fieldsFormat;
   }
+
+  @Override
+  public DocValuesFormat docValuesFormat() {
+    return docValues;
+  }
+
+  @Override
+  public NormsFormat normsFormat() {
+    return norms;
+  }
 }