LUCENE-3312: Apply lucene-3312-patch-07.patch

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3312@1359139 13f79535-47bb-0310-9956-ffa450edef68

lucene/core/src/java/org/apache/lucene/document/Document.java

@@ -28,8 +28,6 @@ import org.apache.lucene.search.ScoreDoc; // for javadoc
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FilterIterator;
 
-import com.google.common.collect.AbstractIterator;
-
 /** Documents are the unit of indexing and search.
  *
  * A Document is a set of fields.  Each field has a name and a textual value.
@@ -66,6 +64,14 @@ public final class Document implements IndexDocument{
     fields.add(field);
   }
   
+  public final void add(IndexableField field) {
+    fields.add((Field) field);
+  }
+  
+  public final void add(StorableField field) {
+    fields.add((Field) field);
+  }
+  
   /**
    * <p>Removes field with the specified name from the document.
    * If multiple fields exist with this name, this method removes the first field that has been added.

lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java

@@ -62,15 +62,12 @@ public class DocumentStoredFieldVisitor extends StoredFieldVisitor {
 
   @Override
   public void stringField(FieldInfo fieldInfo, String value) throws IOException {
-    /*
     final FieldType ft = new FieldType(TextField.TYPE_STORED);
     ft.setStoreTermVectors(fieldInfo.hasVectors());
     ft.setIndexed(fieldInfo.isIndexed());
     ft.setOmitNorms(fieldInfo.omitsNorms());
     ft.setIndexOptions(fieldInfo.getIndexOptions());
-    */
+    doc.add(new StoredField(fieldInfo.name, value, ft));
-    doc.add(new StoredField(fieldInfo.name, value));
-    //doc.add(new Field(fieldInfo.name, value, ft));
   }
 
   @Override

lucene/core/src/java/org/apache/lucene/document/DoubleField.java

@@ -116,6 +116,7 @@ public final class DoubleField extends Field {
   
   public static final FieldType TYPE_NOT_STORED = new FieldType();
   static {
+    TYPE_NOT_STORED.setIndexed(true);
     TYPE_NOT_STORED.setTokenized(true);
     TYPE_NOT_STORED.setOmitNorms(true);
     TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);

lucene/core/src/java/org/apache/lucene/document/FieldType.java

@@ -20,14 +20,13 @@ package org.apache.lucene.document;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.IndexableFieldType;
-import org.apache.lucene.index.StorableFieldType;
 import org.apache.lucene.search.NumericRangeQuery; // javadocs
 import org.apache.lucene.util.NumericUtils;
 
 /**
  * Describes the properties of a field.
  */
-public class FieldType implements IndexableFieldType, StorableFieldType {
+public class FieldType implements IndexableFieldType  {
 
   /** Data type of the numeric value
    * @since 3.2
@@ -240,5 +239,6 @@ public class FieldType implements IndexableFieldType, StorableFieldType {
   public void setDocValueType(DocValues.Type type) {
     checkIfFrozen();
     docValueType = type;
+    this.stored = true;
   }
 }

lucene/core/src/java/org/apache/lucene/document/FloatField.java

@@ -116,6 +116,7 @@ public final class FloatField extends Field {
   
   public static final FieldType TYPE_NOT_STORED = new FieldType();
   static {
+    TYPE_NOT_STORED.setIndexed(true);
     TYPE_NOT_STORED.setTokenized(true);
     TYPE_NOT_STORED.setOmitNorms(true);
     TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);

lucene/core/src/java/org/apache/lucene/document/IntField.java

@@ -116,6 +116,7 @@ public final class IntField extends Field {
   
   public static final FieldType TYPE_NOT_STORED = new FieldType();
   static {
+    TYPE_NOT_STORED.setIndexed(true);
     TYPE_NOT_STORED.setTokenized(true);
     TYPE_NOT_STORED.setOmitNorms(true);
     TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);

lucene/core/src/java/org/apache/lucene/document/PackedLongDocValuesField.java

@@ -44,6 +44,7 @@ public class PackedLongDocValuesField extends StoredField {
   public static final FieldType TYPE = new FieldType();
   static {
     TYPE.setDocValueType(DocValues.Type.VAR_INTS);
+    TYPE.setStored(true);
     TYPE.freeze();
   }
 

lucene/core/src/java/org/apache/lucene/document/StoredDocument.java

@@ -4,6 +4,7 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.StorableField;
 import org.apache.lucene.util.BytesRef;
 
@@ -183,7 +184,7 @@ public class StoredDocument implements Iterable<StorableField>{
     Document doc = new Document();
     
     for (StorableField field : fields) {
-      Field newField = new Field(field.name(), field.fieldType());
+      Field newField = new Field(field.name(), (FieldType) field.fieldType());
       
       newField.fieldsData = field.stringValue();
       if (newField.fieldsData == null) 
@@ -198,4 +199,19 @@ public class StoredDocument implements Iterable<StorableField>{
     
     return doc;
   }
+
+  /** Prints the fields of a document for human consumption. */
+  @Override
+  public final String toString() {
+    StringBuilder buffer = new StringBuilder();
+    buffer.append("StoredDocument<");
+    for (int i = 0; i < fields.size(); i++) {
+      StorableField field = fields.get(i);
+      buffer.append(field.toString());
+      if (i != fields.size()-1)
+        buffer.append(" ");
+    }
+    buffer.append(">");
+    return buffer.toString();
+  }
 }

lucene/core/src/java/org/apache/lucene/document/StoredField.java

@@ -35,12 +35,10 @@ public class StoredField extends Field {
 
   protected StoredField(String name, FieldType type) {
     super(name, type);
-    this.type.setStored(true);
   }
   
   public StoredField(String name, BytesRef bytes, FieldType type) {
     super(name, bytes, type);
-    this.type.setStored(true);
   }
   
   public StoredField(String name, byte[] value) {
@@ -59,6 +57,10 @@ public class StoredField extends Field {
     super(name, value, TYPE);
   }
   
+  public StoredField(String name, String value, FieldType type) {
+    super(name, value, type);
+  }
+
   public StoredField(String name, int value) {
     super(name, TYPE);
     fieldsData = value;

lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java

@@ -28,6 +28,7 @@ import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.FieldInfosWriter;
 import org.apache.lucene.codecs.PerDocConsumer;
+import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
 import org.apache.lucene.index.TypePromoter.TypeCompatibility;
 import org.apache.lucene.store.IOContext;
@@ -220,62 +221,18 @@ final class DocFieldProcessor extends DocConsumer {
 
     for(IndexableField field : docState.doc.indexableFields()) {
       final String fieldName = field.name();
+      IndexableFieldType ft = field.fieldType();
 
-      // Make sure we have a PerField allocated
+      DocFieldProcessorPerField fp = processField(fieldInfos, thisFieldGen, fieldName, ft);
-      final int hashPos = fieldName.hashCode() & hashMask;
-      DocFieldProcessorPerField fp = fieldHash[hashPos];
-      while(fp != null && !fp.fieldInfo.name.equals(fieldName)) {
-        fp = fp.next;
-      }
-
-      if (fp == null) {
-
-        // TODO FI: we need to genericize the "flags" that a
-        // field holds, and, how these flags are merged; it
-        // needs to be more "pluggable" such that if I want
-        // to have a new "thing" my Fields can do, I can
-        // easily add it
-        FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.fieldType());
-
-        fp = new DocFieldProcessorPerField(this, fi);
-        fp.next = fieldHash[hashPos];
-        fieldHash[hashPos] = fp;
-        totalFieldCount++;
-
-        if (totalFieldCount >= fieldHash.length/2) {
-          rehash();
-        }
-      } else {
-        fieldInfos.addOrUpdate(fp.fieldInfo.name, field.fieldType());
-      }
-
-      if (thisFieldGen != fp.lastGen) {
-
-        // First time we're seeing this field for this doc
-        fp.fieldCount = 0;
-
-        if (fieldCount == fields.length) {
-          final int newSize = fields.length*2;
-          DocFieldProcessorPerField newArray[] = new DocFieldProcessorPerField[newSize];
-          System.arraycopy(fields, 0, newArray, 0, fieldCount);
-          fields = newArray;
-        }
-
-        fields[fieldCount++] = fp;
-        fp.lastGen = thisFieldGen;
-      }
 
       fp.addField(field);
     }
     for (StorableField field: docState.doc.storableFields()) {
       final String fieldName = field.name();
+      IndexableFieldType ft = field.fieldType();
 
-      // Make sure we have a PerField allocated
+      DocFieldProcessorPerField fp = processField(fieldInfos, thisFieldGen, fieldName, ft);
-      final int hashPos = fieldName.hashCode() & hashMask;
+      fieldsWriter.addField(field, fp.fieldInfo);
-      DocFieldProcessorPerField fp = fieldHash[hashPos];
-      while(fp != null && !fp.fieldInfo.name.equals(fieldName)) {
-        fp = fp.next;
-      }
       
       final DocValues.Type dvType = field.fieldType().docValueType();
       if (dvType != null) {
@@ -320,6 +277,54 @@ final class DocFieldProcessor extends DocConsumer {
     }
   }
 
+  private DocFieldProcessorPerField processField(FieldInfos.Builder fieldInfos,
+      final int thisFieldGen, final String fieldName, IndexableFieldType ft) {
+    // Make sure we have a PerField allocated
+    final int hashPos = fieldName.hashCode() & hashMask;
+    DocFieldProcessorPerField fp = fieldHash[hashPos];
+    while(fp != null && !fp.fieldInfo.name.equals(fieldName)) {
+      fp = fp.next;
+    }
+
+    if (fp == null) {
+
+      // TODO FI: we need to genericize the "flags" that a
+      // field holds, and, how these flags are merged; it
+      // needs to be more "pluggable" such that if I want
+      // to have a new "thing" my Fields can do, I can
+      // easily add it
+      FieldInfo fi = fieldInfos.addOrUpdate(fieldName, ft);
+
+      fp = new DocFieldProcessorPerField(this, fi);
+      fp.next = fieldHash[hashPos];
+      fieldHash[hashPos] = fp;
+      totalFieldCount++;
+
+      if (totalFieldCount >= fieldHash.length/2) {
+        rehash();
+      }
+    } else {
+      fieldInfos.addOrUpdate(fp.fieldInfo.name, ft);
+    }
+
+    if (thisFieldGen != fp.lastGen) {
+
+      // First time we're seeing this field for this doc
+      fp.fieldCount = 0;
+
+      if (fieldCount == fields.length) {
+        final int newSize = fields.length*2;
+        DocFieldProcessorPerField newArray[] = new DocFieldProcessorPerField[newSize];
+        System.arraycopy(fields, 0, newArray, 0, fieldCount);
+        fields = newArray;
+      }
+
+      fields[fieldCount++] = fp;
+      fp.lastGen = thisFieldGen;
+    }
+    return fp;
+  }
+
   private static final Comparator<DocFieldProcessorPerField> fieldsComp = new Comparator<DocFieldProcessorPerField>() {
     public int compare(DocFieldProcessorPerField o1, DocFieldProcessorPerField o2) {
       return o1.fieldInfo.name.compareTo(o2.fieldInfo.name);

lucene/core/src/java/org/apache/lucene/index/FieldInfos.java

@@ -260,7 +260,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
       // rather, each component in the chain should update
       // what it "owns".  EG fieldType.indexOptions() should
       // be updated by maybe FreqProxTermsWriterPerField:
-      return addOrUpdateInternal(name, -1, true, false,
+      return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
                                  fieldType.omitNorms(), false,
                                  fieldType.indexOptions(), null, null);
     }

lucene/core/src/java/org/apache/lucene/index/GeneralField.java

@@ -0,0 +1,44 @@
+package org.apache.lucene.index;
+
+import java.io.Reader;
+
+import org.apache.lucene.util.BytesRef;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public interface GeneralField {
+
+  /** Field name */
+  public String name();
+
+  /** {@link IndexableFieldType} describing the properties
+   * of this field. */
+  public IndexableFieldType fieldType();
+
+  /** Non-null if this field has a binary value */
+  public BytesRef binaryValue();
+
+  /** Non-null if this field has a string value */
+  public String stringValue();
+
+  /** Non-null if this field has a Reader value */
+  public Reader readerValue();
+
+  /** Non-null if this field has a numeric value */
+  public Number numericValue();
+}

lucene/core/src/java/org/apache/lucene/index/IndexableField.java

@@ -33,14 +33,7 @@ import org.apache.lucene.util.BytesRef;
  *
  *  @lucene.experimental */
 
-public interface IndexableField {
+public interface IndexableField extends GeneralField {
-
-  /** Field name */
-  public String name();
-  
-  /** {@link IndexableFieldType} describing the properties
-   * of this field. */
-  public IndexableFieldType fieldType();
 
   /**
    * Creates the TokenStream used for indexing this field.  If appropriate,

lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java

@@ -25,6 +25,12 @@ import org.apache.lucene.index.FieldInfo.IndexOptions;
  */
 public interface IndexableFieldType {
 
+  /** True if this field should be indexed (inverted) */
+  public boolean indexed();
+
+  /** True if the field's value should be stored */
+  public boolean stored();
+  
   /** True if this field's value should be analyzed */
   public boolean tokenized();
 
@@ -43,4 +49,8 @@ public interface IndexableFieldType {
   /** {@link IndexOptions}, describing what should be
    * recorded into the inverted index */
   public IndexOptions indexOptions();
+
+  /** DocValues type; if non-null then the field's value
+   *  will be indexed into docValues */
+  public DocValues.Type docValueType();  
 }

lucene/core/src/java/org/apache/lucene/index/StorableField.java

@@ -22,23 +22,6 @@ import org.apache.lucene.util.BytesRef;
  * limitations under the License.
  */
 
-public interface StorableField {
+public interface StorableField extends GeneralField {
   
-  /** Field name */
-  public String name();
-  
-  /** Field type */
-  public FieldType fieldType();
-  
-  /** Non-null if this field has a binary value */
-  public BytesRef binaryValue();
-
-  /** Non-null if this field has a string value */
-  public String stringValue();
-
-  /** Non-null if this field has a Reader value */
-  public Reader readerValue();
-
-  /** Non-null if this field has a numeric value */
-  public Number numericValue();
 }

lucene/core/src/test/org/apache/lucene/TestDemo.java

@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
@@ -64,7 +65,7 @@ public class TestDemo extends LuceneTestCase {
     assertEquals(1, hits.totalHits);
     // Iterate through the results:
     for (int i = 0; i < hits.scoreDocs.length; i++) {
-      Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
+      StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
       assertEquals(text, hitDoc.get("fieldname"));
     }
 

lucene/core/src/test/org/apache/lucene/TestSearch.java

@@ -110,7 +110,7 @@ public class TestSearch extends LuceneTestCase {
 
         out.println(hits.length + " total results");
         for (int i = 0 ; i < hits.length && i < 10; i++) {
-          Document d = searcher.doc(hits[i].doc);
+          StoredDocument d = searcher.doc(hits[i].doc);
           out.println(i + " " + hits[i].score + " " + d.get("contents"));
         }
       }

lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java

@@ -127,7 +127,7 @@ public class TestSearchForDuplicates extends LuceneTestCase {
     out.println(hits.length + " total results\n");
     for (int i = 0 ; i < hits.length; i++) {
       if ( i < 10 || (i > 94 && i < 105) ) {
-        Document d = searcher.doc(hits[i].doc);
+        StoredDocument d = searcher.doc(hits[i].doc);
         out.println(i + " " + d.get(ID_FIELD));
       }
     }
@@ -137,7 +137,7 @@ public class TestSearchForDuplicates extends LuceneTestCase {
     assertEquals("total results", expectedCount, hits.length);
     for (int i = 0 ; i < hits.length; i++) {
       if (i < 10 || (i > 94 && i < 105) ) {
-        Document d = searcher.doc(hits[i].doc);
+        StoredDocument d = searcher.doc(hits[i].doc);
         assertEquals("check " + i, String.valueOf(i), d.get(ID_FIELD));
       }
     }

lucene/core/src/test/org/apache/lucene/codecs/appending/TestAppendingCodec.java

@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.appending.AppendingCodec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocsEnum;
@@ -128,7 +129,7 @@ public class TestAppendingCodec extends LuceneTestCase {
     writer.close();
     IndexReader reader = DirectoryReader.open(dir, 1);
     assertEquals(2, reader.numDocs());
-    Document doc2 = reader.document(0);
+    StoredDocument doc2 = reader.document(0);
     assertEquals(text, doc2.get("f"));
     Fields fields = MultiFields.getFields(reader);
     Terms terms = fields.terms("f");

lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java

@@ -28,12 +28,14 @@ import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.lucene40.values.Bytes;
 import org.apache.lucene.codecs.lucene40.values.Floats;
 import org.apache.lucene.codecs.lucene40.values.Ints;
+import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.DocValues.SortedSource;
 import org.apache.lucene.index.DocValues.Source;
 import org.apache.lucene.index.DocValues.Type;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
+import org.apache.lucene.index.StorableField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
@@ -438,21 +440,11 @@ public class TestDocValues extends LuceneTestCase {
     return getSource(values).asSortedSource();
   }
   
-  public static class DocValueHolder implements IndexableField {
+  public static class DocValueHolder implements StorableField {
     BytesRef bytes;
     Number numberValue;
     Comparator<BytesRef> comp;
 
-    @Override
-    public TokenStream tokenStream(Analyzer a) {
-      return null;
-    }
-
-    @Override
-    public float boost() {
-      return 0.0f;
-    }
-
     @Override
     public String name() {
       return "test";
@@ -479,7 +471,7 @@ public class TestDocValues extends LuceneTestCase {
     }
 
     @Override
-    public IndexableFieldType fieldType() {
+    public FieldType fieldType() {
       return null;
     }
   }

lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java

@@ -37,8 +37,8 @@ public class TestBinaryDocument extends LuceneTestCase {
   {
     FieldType ft = new FieldType();
     ft.setStored(true);
-    IndexableField binaryFldStored = new StoredField("binaryStored", binaryValStored.getBytes());
+    StoredField binaryFldStored = new StoredField("binaryStored", binaryValStored.getBytes());
-    IndexableField stringFldStored = new Field("stringStored", binaryValStored, ft);
+    Field stringFldStored = new Field("stringStored", binaryValStored, ft);
 
     Document doc = new Document();
     
@@ -56,7 +56,7 @@ public class TestBinaryDocument extends LuceneTestCase {
     
     /** open a reader and fetch the document */ 
     IndexReader reader = writer.getReader();
-    Document docFromReader = reader.document(0);
+    StoredDocument docFromReader = reader.document(0);
     assertTrue(docFromReader != null);
     
     /** fetch the binary stored field and compare it's content with the original one */
@@ -75,8 +75,8 @@ public class TestBinaryDocument extends LuceneTestCase {
   }
   
   public void testCompressionTools() throws Exception {
-    IndexableField binaryFldCompressed = new StoredField("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes()));
+    StoredField binaryFldCompressed = new StoredField("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes()));
-    IndexableField stringFldCompressed = new StoredField("stringCompressed", CompressionTools.compressString(binaryValCompressed));
+    StoredField stringFldCompressed = new StoredField("stringCompressed", CompressionTools.compressString(binaryValCompressed));
     
     Document doc = new Document();
     
@@ -90,7 +90,7 @@ public class TestBinaryDocument extends LuceneTestCase {
     
     /** open a reader and fetch the document */ 
     IndexReader reader = writer.getReader();
-    Document docFromReader = reader.document(0);
+    StoredDocument docFromReader = reader.document(0);
     assertTrue(docFromReader != null);
     
     /** fetch the binary compressed field and compare it's content with the original one */

lucene/core/src/test/org/apache/lucene/document/TestDocument.java

@@ -50,9 +50,9 @@ public class TestDocument extends LuceneTestCase {
     
     FieldType ft = new FieldType();
     ft.setStored(true);
-    IndexableField stringFld = new Field("string", binaryVal, ft);
+    Field stringFld = new Field("string", binaryVal, ft);
-    IndexableField binaryFld = new StoredField("binary", binaryVal.getBytes("UTF-8"));
+    StoredField binaryFld = new StoredField("binary", binaryVal.getBytes("UTF-8"));
-    IndexableField binaryFld2 = new StoredField("binary", binaryVal2.getBytes("UTF-8"));
+    StoredField binaryFld2 = new StoredField("binary", binaryVal2.getBytes("UTF-8"));
     
     doc.add(stringFld);
     doc.add(binaryFld);
@@ -179,7 +179,7 @@ public class TestDocument extends LuceneTestCase {
     ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
     assertEquals(1, hits.length);
     
-    doAssert(searcher.doc(hits[0].doc), true);
+    doAssert(searcher.doc(hits[0].doc));
     writer.close();
     reader.close();
     dir.close();
@@ -214,6 +214,9 @@ public class TestDocument extends LuceneTestCase {
     return doc;
   }
   
+  private void doAssert(StoredDocument doc) {
+    doAssert(doc.asIndexable(), true);
+  }
   private void doAssert(Document doc, boolean fromIndex) {
     IndexableField[] keywordFieldValues = doc.getFields("keyword");
     IndexableField[] textFieldValues = doc.getFields("text");
@@ -268,7 +271,7 @@ public class TestDocument extends LuceneTestCase {
     assertEquals(3, hits.length);
     int result = 0;
     for (int i = 0; i < 3; i++) {
-      Document doc2 = searcher.doc(hits[i].doc);
+      StoredDocument doc2 = searcher.doc(hits[i].doc);
       Field f = (Field) doc2.getField("id");
       if (f.stringValue().equals("id1")) result |= 1;
       else if (f.stringValue().equals("id2")) result |= 2;

lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java

@@ -45,6 +45,7 @@ import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -1245,7 +1246,7 @@ public class TestAddIndexes extends LuceneTestCase {
     w.close();
     assertEquals(2, r3.numDocs());
     for(int docID=0;docID<2;docID++) {
-      Document d = r3.document(docID);
+      StoredDocument d = r3.document(docID);
       if (d.get("id").equals("1")) {
         assertEquals("doc1 field1", d.get("f1"));
       } else {

lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -43,6 +43,7 @@ import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.PackedLongDocValuesField;
 import org.apache.lucene.document.ShortDocValuesField;
 import org.apache.lucene.document.SortedBytesDocValuesField;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StraightBytesDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
@@ -328,13 +329,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
 
     for(int i=0;i<35;i++) {
       if (liveDocs.get(i)) {
-        Document d = reader.document(i);
+        StoredDocument d = reader.document(i);
-        List<IndexableField> fields = d.getFields();
+        List<StorableField> fields = d.getFields();
         boolean isProxDoc = d.getField("content3") == null;
         if (isProxDoc) {
           final int numFields = is40Index ? 7 : 5;
           assertEquals(numFields, fields.size());
-          IndexableField f =  d.getField("id");
+          StorableField f =  d.getField("id");
           assertEquals(""+i, f.stringValue());
 
           f = d.getField("utf8");
@@ -405,7 +406,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     ScoreDoc[] hits = searcher.search(new TermQuery(new Term(new String("content"), "aaa")), null, 1000).scoreDocs;
 
     // First document should be #0
-    Document d = searcher.getIndexReader().document(hits[0].doc);
+    StoredDocument d = searcher.getIndexReader().document(hits[0].doc);
     assertEquals("didn't get the right document first", "0", d.get("id"));
 
     doTestHits(hits, 34, searcher.getIndexReader());
@@ -458,7 +459,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     IndexReader reader = DirectoryReader.open(dir);
     IndexSearcher searcher = new IndexSearcher(reader);
     ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
-    Document d = searcher.getIndexReader().document(hits[0].doc);
+    StoredDocument d = searcher.getIndexReader().document(hits[0].doc);
     assertEquals("wrong first document", "0", d.get("id"));
     doTestHits(hits, 44, searcher.getIndexReader());
     reader.close();
@@ -484,7 +485,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     IndexSearcher searcher = new IndexSearcher(reader);
     ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
     assertEquals("wrong number of hits", 34, hits.length);
-    Document d = searcher.doc(hits[0].doc);
+    StoredDocument d = searcher.doc(hits[0].doc);
     assertEquals("wrong first document", "0", d.get("id"));
     reader.close();
 
@@ -752,7 +753,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
       for (int id=10; id<15; id++) {
         ScoreDoc[] hits = searcher.search(NumericRangeQuery.newIntRange("trieInt", 4, Integer.valueOf(id), Integer.valueOf(id), true, true), 100).scoreDocs;
         assertEquals("wrong number of hits", 1, hits.length);
-        Document d = searcher.doc(hits[0].doc);
+        StoredDocument d = searcher.doc(hits[0].doc);
         assertEquals(String.valueOf(id), d.get("id"));
         
         hits = searcher.search(NumericRangeQuery.newLongRange("trieLong", 4, Long.valueOf(id), Long.valueOf(id), true, true), 100).scoreDocs;

lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java

@@ -22,6 +22,7 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DocValues.Source;
 import org.apache.lucene.index.DocValues.Type;
@@ -75,7 +76,7 @@ public class TestCustomNorms extends LuceneTestCase {
     assertEquals(Type.FLOAT_32, normValues.getType());
     float[] norms = (float[]) source.getArray();
     for (int i = 0; i < open.maxDoc(); i++) {
-      Document document = open.document(i);
+      StoredDocument document = open.document(i);
       float expected = Float.parseFloat(document.get(floatTestField));
       assertEquals(expected, norms[i], 0.0f);
     }

lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java

@@ -32,6 +32,7 @@ import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
@@ -61,10 +62,10 @@ public class TestDirectoryReader extends LuceneTestCase {
     assertTrue(reader != null);
     assertTrue(reader instanceof StandardDirectoryReader);
     
-    Document newDoc1 = reader.document(0);
+    StoredDocument newDoc1 = reader.document(0);
     assertTrue(newDoc1 != null);
     assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
-    Document newDoc2 = reader.document(1);
+    StoredDocument newDoc2 = reader.document(1);
     assertTrue(newDoc2 != null);
     assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
     Terms vector = reader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
@@ -386,11 +387,11 @@ void assertTermDocsCount(String msg,
       writer.addDocument(doc);
       writer.close();
       DirectoryReader reader = DirectoryReader.open(dir);
-      Document doc2 = reader.document(reader.maxDoc() - 1);
+      StoredDocument doc2 = reader.document(reader.maxDoc() - 1);
-      IndexableField[] fields = doc2.getFields("bin1");
+      StorableField[] fields = doc2.getFields("bin1");
       assertNotNull(fields);
       assertEquals(1, fields.length);
-      IndexableField b1 = fields[0];
+      StorableField b1 = fields[0];
       assertTrue(b1.binaryValue() != null);
       BytesRef bytesRef = b1.binaryValue();
       assertEquals(bin.length, bytesRef.length);
@@ -595,13 +596,13 @@ public void testFilesOpenClose() throws IOException {
     // check stored fields
     for (int i = 0; i < index1.maxDoc(); i++) {
       if (liveDocs1 == null || liveDocs1.get(i)) {
-        Document doc1 = index1.document(i);
+        StoredDocument doc1 = index1.document(i);
-        Document doc2 = index2.document(i);
+        StoredDocument doc2 = index2.document(i);
-        List<IndexableField> field1 = doc1.getFields();
+        List<StorableField> field1 = doc1.getFields();
-        List<IndexableField> field2 = doc2.getFields();
+        List<StorableField> field2 = doc2.getFields();
         assertEquals("Different numbers of fields for doc " + i + ".", field1.size(), field2.size());
-        Iterator<IndexableField> itField1 = field1.iterator();
+        Iterator<StorableField> itField1 = field1.iterator();
-        Iterator<IndexableField> itField2 = field2.iterator();
+        Iterator<StorableField> itField2 = field2.iterator();
         while (itField1.hasNext()) {
           Field curField1 = (Field) itField1.next();
           Field curField2 = (Field) itField2.next();
@@ -1080,7 +1081,7 @@ public void testFilesOpenClose() throws IOException {
     Set<String> fieldsToLoad = new HashSet<String>();
     assertEquals(0, r.document(0, fieldsToLoad).getFields().size());
     fieldsToLoad.add("field1");
-    Document doc2 = r.document(0, fieldsToLoad);
+    StoredDocument doc2 = r.document(0, fieldsToLoad);
     assertEquals(1, doc2.getFields().size());
     assertEquals("foobar", doc2.get("field1"));
     r.close();

lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java

@@ -32,6 +32,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.search.IndexSearcher;
@@ -122,7 +123,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
           if (i>0) {
             int k = i-1;
             int n = j + k*M;
-            Document prevItereationDoc = reader.document(n);
+            StoredDocument prevItereationDoc = reader.document(n);
             assertNotNull(prevItereationDoc);
             String id = prevItereationDoc.get("id");
             assertEquals(k+"_"+j, id);

lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java

@@ -122,7 +122,7 @@ public class TestDocTermOrds extends LuceneTestCase {
     for(int id=0;id<NUM_DOCS;id++) {
       Document doc = new Document();
 
-      doc.add(new IntField("id", id, Field.Store.NO));
+      doc.add(new IntField("id", id, Field.Store.YES));
       
       final int termCount = _TestUtil.nextInt(random(), 0, 20*RANDOM_MULTIPLIER);
       while(ordsForDocSet.size() < termCount) {
@@ -219,7 +219,7 @@ public class TestDocTermOrds extends LuceneTestCase {
     for(int id=0;id<NUM_DOCS;id++) {
       Document doc = new Document();
 
-      doc.add(new IntField("id", id, Field.Store.NO));
+      doc.add(new IntField("id", id, Field.Store.YES));
       
       final int termCount = _TestUtil.nextInt(random(), 0, 20*RANDOM_MULTIPLIER);
       while(ordsForDocSet.size() < termCount) {

lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java

@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -67,11 +68,11 @@ public class TestDocumentWriter extends LuceneTestCase {
     //After adding the document, we should be able to read it back in
     SegmentReader reader = new SegmentReader(info, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random()));
     assertTrue(reader != null);
-    Document doc = reader.document(0);
+    StoredDocument doc = reader.document(0);
     assertTrue(doc != null);
 
     //System.out.println("Document: " + doc);
-    IndexableField [] fields = doc.getFields("textField2");
+    StorableField[] fields = doc.getFields("textField2");
     assertTrue(fields != null && fields.length == 1);
     assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_2_TEXT));
     assertTrue(fields[0].fieldType().storeTermVectors());

lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java

@@ -31,6 +31,7 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
@@ -548,25 +549,25 @@ public class TestDuelingCodecs extends LuceneTestCase {
   public void assertStoredFields(IndexReader leftReader, IndexReader rightReader) throws Exception {
     assert leftReader.maxDoc() == rightReader.maxDoc();
     for (int i = 0; i < leftReader.maxDoc(); i++) {
-      Document leftDoc = leftReader.document(i);
+      StoredDocument leftDoc = leftReader.document(i);
-      Document rightDoc = rightReader.document(i);
+      StoredDocument rightDoc = rightReader.document(i);
       
       // TODO: I think this is bogus because we don't document what the order should be
       // from these iterators, etc. I think the codec/IndexReader should be free to order this stuff
       // in whatever way it wants (e.g. maybe it packs related fields together or something)
       // To fix this, we sort the fields in both documents by name, but
       // we still assume that all instances with same name are in order:
-      Comparator<IndexableField> comp = new Comparator<IndexableField>() {
+      Comparator<StorableField> comp = new Comparator<StorableField>() {
         @Override
-        public int compare(IndexableField arg0, IndexableField arg1) {
+        public int compare(StorableField arg0, StorableField arg1) {
           return arg0.name().compareTo(arg1.name());
         }        
       };
       Collections.sort(leftDoc.getFields(), comp);
       Collections.sort(rightDoc.getFields(), comp);
 
-      Iterator<IndexableField> leftIterator = leftDoc.iterator();
+      Iterator<StorableField> leftIterator = leftDoc.iterator();
-      Iterator<IndexableField> rightIterator = rightDoc.iterator();
+      Iterator<StorableField> rightIterator = rightDoc.iterator();
       while (leftIterator.hasNext()) {
         assertTrue(info, rightIterator.hasNext());
         assertStoredField(leftIterator.next(), rightIterator.next());
@@ -578,7 +579,7 @@ public class TestDuelingCodecs extends LuceneTestCase {
   /** 
    * checks that two stored fields are equivalent 
    */
-  public void assertStoredField(IndexableField leftField, IndexableField rightField) {
+  public void assertStoredField(StorableField leftField, StorableField rightField) {
     assertEquals(info, leftField.name(), rightField.name());
     assertEquals(info, leftField.binaryValue(), rightField.binaryValue());
     assertEquals(info, leftField.stringValue(), rightField.stringValue());

lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java

@@ -47,7 +47,7 @@ public class TestFieldInfos extends LuceneTestCase {
   //Positive test of FieldInfos
     assertTrue(testDoc != null);
     FieldInfos.Builder builder = new FieldInfos.Builder();
-    for (IndexableField field : testDoc) {
+    for (IndexableField field : testDoc.getFields()) {
       builder.addOrUpdate(field.name(), field.fieldType());
     }
     FieldInfos fieldInfos = builder.finish();

lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java

@@ -31,6 +31,7 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.FloatField;
 import org.apache.lucene.document.IntField;
 import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -55,7 +56,7 @@ public class TestFieldsReader extends LuceneTestCase {
   public static void beforeClass() throws Exception {
     fieldInfos = new FieldInfos.Builder();
     DocHelper.setupDoc(testDoc);
-    for (IndexableField field : testDoc) {
+    for (IndexableField field : testDoc.getFields()) {
       fieldInfos.addOrUpdate(field.name(), field.fieldType());
     }
     dir = newDirectory();
@@ -79,7 +80,7 @@ public class TestFieldsReader extends LuceneTestCase {
     assertTrue(dir != null);
     assertTrue(fieldInfos != null);
     IndexReader reader = DirectoryReader.open(dir);
-    Document doc = reader.document(0);
+    StoredDocument doc = reader.document(0);
     assertTrue(doc != null);
     assertTrue(doc.getField(DocHelper.TEXT_FIELD_1_KEY) != null);
 
@@ -104,7 +105,7 @@ public class TestFieldsReader extends LuceneTestCase {
 
     DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(DocHelper.TEXT_FIELD_3_KEY);
     reader.document(0, visitor);
-    final List<IndexableField> fields = visitor.getDocument().getFields();
+    final List<StorableField> fields = visitor.getDocument().getFields();
     assertEquals(1, fields.size());
     assertEquals(DocHelper.TEXT_FIELD_3_KEY, fields.get(0).name());
     reader.close();
@@ -279,7 +280,7 @@ public class TestFieldsReader extends LuceneTestCase {
       doc.add(sf);
       answers[id] = answer;
       typeAnswers[id] = typeAnswer;
-      FieldType ft = new FieldType(IntField.TYPE_NOT_STORED);
+      FieldType ft = new FieldType(IntField.TYPE_STORED);
       ft.setNumericPrecisionStep(Integer.MAX_VALUE);
       doc.add(new IntField("id", id, ft));
       w.addDocument(doc);
@@ -292,7 +293,7 @@ public class TestFieldsReader extends LuceneTestCase {
     for(IndexReader sub : r.getSequentialSubReaders()) {
       final int[] ids = FieldCache.DEFAULT.getInts((AtomicReader) sub, "id", false);
       for(int docID=0;docID<sub.numDocs();docID++) {
-        final Document doc = sub.document(docID);
+        final StoredDocument doc = sub.document(docID);
         final Field f = (Field) doc.getField("nf");
         assertTrue("got f=" + f, f instanceof StoredField);
         assertEquals(answers[ids[docID]], f.numericValue());

lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java

@@ -34,6 +34,7 @@ import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -906,8 +907,8 @@ public class TestIndexWriter extends LuceneTestCase {
     w.close();
 
     IndexReader ir = DirectoryReader.open(dir);
-    Document doc2 = ir.document(0);
+    StoredDocument doc2 = ir.document(0);
-    IndexableField f2 = doc2.getField("binary");
+    StorableField f2 = doc2.getField("binary");
     b = f2.binaryValue().bytes;
     assertTrue(b != null);
     assertEquals(17, b.length, 17);
@@ -1163,8 +1164,8 @@ public class TestIndexWriter extends LuceneTestCase {
     w.close();
 
     IndexReader ir = DirectoryReader.open(dir);
-    Document doc2 = ir.document(0);
+    StoredDocument doc2 = ir.document(0);
-    IndexableField f3 = doc2.getField("binary");
+    StorableField f3 = doc2.getField("binary");
     b = f3.binaryValue().bytes;
     assertTrue(b != null);
     assertEquals(17, b.length, 17);
@@ -1205,8 +1206,8 @@ public class TestIndexWriter extends LuceneTestCase {
     doc.add(newField("zzz", "1 2 3", customType));
     w.addDocument(doc);
     IndexReader r = w.getReader();
-    Document doc2 = r.document(0);
+    StoredDocument doc2 = r.document(0);
-    Iterator<IndexableField> it = doc2.getFields().iterator();
+    Iterator<StorableField> it = doc2.getFields().iterator();
     assertTrue(it.hasNext());
     Field f = (Field) it.next();
     assertEquals(f.name(), "zzz");

lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java

@@ -21,6 +21,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.util.LuceneTestCase;
@@ -84,7 +85,7 @@ public class TestIndexWriterMerging extends LuceneTestCase
     int max = reader.maxDoc();
     for (int i = 0; i < max; i++)
     {
-      Document temp = reader.document(i);
+      StoredDocument temp = reader.document(i);
       //System.out.println("doc "+i+"="+temp.getField("count").stringValue());
       //compare the index doc number to the value that it should be
       if (!temp.getField("count").stringValue().equals((i + startAt) + ""))

lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java

@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
@@ -141,10 +142,10 @@ public class TestIndexWriterReader extends LuceneTestCase {
 
     String id10 = r1.document(10).getField("id").stringValue();
     
-    Document newDoc = r1.document(10);
+    StoredDocument newDoc = r1.document(10);
     newDoc.removeField("id");
     newDoc.add(newStringField("id", Integer.toString(8000), Field.Store.YES));
-    writer.updateDocument(new Term("id", id10), newDoc);
+    writer.updateDocument(new Term("id", id10), newDoc.asIndexable());
     assertFalse(r1.isCurrent());
 
     DirectoryReader r2 = writer.getReader();
@@ -271,9 +272,9 @@ public class TestIndexWriterReader extends LuceneTestCase {
     assertEquals(100, index2df);
 
     // verify the docs are from different indexes
-    Document doc5 = r1.document(5);
+    StoredDocument doc5 = r1.document(5);
     assertEquals("index1", doc5.get("indexname"));
-    Document doc150 = r1.document(150);
+    StoredDocument doc150 = r1.document(150);
     assertEquals("index2", doc150.get("indexname"));
     r1.close();
     writer.close();

lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java

@@ -26,6 +26,7 @@ import java.util.Set;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
@@ -259,7 +260,7 @@ public class TestIndexWriterUnicode extends LuceneTestCase {
     w.close();
 
     IndexReader ir = DirectoryReader.open(dir);
-    Document doc2 = ir.document(0);
+    StoredDocument doc2 = ir.document(0);
     for(int i=0;i<count;i++) {
       assertEquals("field " + i + " was not indexed correctly", 1, ir.docFreq(new Term("f"+i, utf8Data[2*i+1])));
       assertEquals("field " + i + " is incorrect", utf8Data[2*i+1], doc2.getField("f"+i).stringValue());

lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java

@@ -24,8 +24,9 @@ import java.util.Iterator;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
+import org.apache.lucene.index.DocValues.Type;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -39,7 +40,7 @@ import org.apache.lucene.util._TestUtil;
 
 public class TestIndexableField extends LuceneTestCase {
 
-  private class MyField implements IndexableField {
+  private class MyField implements IndexableField, StorableField {
 
     private final int counter;
     private final IndexableFieldType fieldType = new IndexableFieldType() {
@@ -84,7 +85,7 @@ public class TestIndexableField extends LuceneTestCase {
       }
 
       @Override
-      public DocValues.Type docValueType() {
+      public Type docValueType() {
         return null;
       }
     };
@@ -178,25 +179,37 @@ public class TestIndexableField extends LuceneTestCase {
       final int finalBaseCount = baseCount;
       baseCount += fieldCount-1;
 
-      w.addDocument(new Iterable<IndexableField>() {
+      IndexDocument d = new IndexDocument() {
+        @Override
+        public Iterable<? extends IndexableField> indexableFields() {
+          return new Iterable<IndexableField>() {
             @Override
             public Iterator<IndexableField> iterator() {
               return new Iterator<IndexableField>() {
-            int fieldUpto;
+                int fieldUpto = 0;
+                private IndexableField next;
 
                 @Override
                 public boolean hasNext() {
-              return fieldUpto < fieldCount;
+                  if (fieldUpto >= fieldCount) return false;
+                  next = null;
+                  if (fieldUpto > 1) 
+                    next = new MyField(finalBaseCount + (fieldUpto++-1));
+                  else 
+                    fieldUpto = 2;
+                  
+                  if (next != null && next.fieldType().indexed()) return true;
+                  else return this.hasNext();
                 }
 
                 @Override
                 public IndexableField next() {
-              assert fieldUpto < fieldCount;
+                  assert fieldUpto <= fieldCount;
-              if (fieldUpto == 0) {
+                  if (next == null && !hasNext()) {
-                fieldUpto = 1;
+                    return null;
-                return newStringField("id", ""+finalDocCount, Field.Store.YES);
+                  }
-              } else {
+                  else {
-                return new MyField(finalBaseCount + (fieldUpto++-1));
+                    return next;
                   }
                 }
 
@@ -206,7 +219,57 @@ public class TestIndexableField extends LuceneTestCase {
                 }
               };
             }
-        });
+          };
+        }
+
+        @Override
+        public Iterable<? extends StorableField> storableFields() {
+          return new Iterable<StorableField>() {
+            @Override
+            public Iterator<StorableField> iterator() {
+              return new Iterator<StorableField>() {
+                int fieldUpto = 0;
+                private StorableField next = null;
+
+                @Override
+                public boolean hasNext() {
+
+                  if (fieldUpto == fieldCount) return false;
+                  
+                  next = null;
+                  if (fieldUpto == 0) {
+                    fieldUpto = 1;
+                    next = newStringField("id", ""+finalDocCount, Field.Store.YES);
+                  } else {
+                    next = new MyField(finalBaseCount + (fieldUpto++-1));
+                  }
+                  
+                  if (next != null && next.fieldType().stored()) return true;
+                  else return this.hasNext();
+                }
+
+                @Override
+                public StorableField next() {
+                  assert fieldUpto <= fieldCount;
+                  if (next == null && !hasNext()) {
+                    return null;
+                  }
+                  else {
+                    return next;
+                  }
+                }
+
+                @Override
+                public void remove() {
+                  throw new UnsupportedOperationException();
+                }
+              };
+            }
+          };
+        }
+      };
+      
+      w.addDocument(d);
     }
 
     final IndexReader r = w.getReader();
@@ -218,10 +281,11 @@ public class TestIndexableField extends LuceneTestCase {
       if (VERBOSE) {
         System.out.println("TEST: verify doc id=" + id + " (" + fieldsPerDoc[id] + " fields) counter=" + counter);
       }
+
       final TopDocs hits = s.search(new TermQuery(new Term("id", ""+id)), 1);
       assertEquals(1, hits.totalHits);
       final int docID = hits.scoreDocs[0].doc;
-      final Document doc = s.doc(docID);
+      final StoredDocument doc = s.doc(docID);
       final int endCounter = counter + fieldsPerDoc[id];
       while(counter < endCounter) {
         final String name = "f" + counter;
@@ -240,7 +304,7 @@ public class TestIndexableField extends LuceneTestCase {
 
         // stored:
         if (stored) {
-          IndexableField f = doc.getField(name);
+          StorableField f = doc.getField(name);
           assertNotNull("doc " + id + " doesn't have field f" + counter, f);
           if (binary) {
             assertNotNull("doc " + id + " doesn't have field f" + counter, f);

lucene/core/src/test/org/apache/lucene/index/TestNorms.java

@@ -23,6 +23,7 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DocValues.Source;
 import org.apache.lucene.index.DocValues.Type;
@@ -105,7 +106,7 @@ public class TestNorms extends LuceneTestCase {
     assertEquals(Type.FIXED_INTS_8, normValues.getType());
     byte[] norms = (byte[]) source.getArray();
     for (int i = 0; i < open.maxDoc(); i++) {
-      Document document = open.document(i);
+      StoredDocument document = open.document(i);
       int expected = Integer.parseInt(document.get(byteTestField));
       assertEquals((byte)expected, norms[i]);
     }
@@ -164,7 +165,7 @@ public class TestNorms extends LuceneTestCase {
       assertEquals(Type.FIXED_INTS_8, normValues.getType());
       byte[] norms = (byte[]) source.getArray();
       for (int i = 0; i < mergedReader.maxDoc(); i++) {
-        Document document = mergedReader.document(i);
+        StoredDocument document = mergedReader.document(i);
         int expected = Integer.parseInt(document.get(byteTestField));
         assertEquals((byte) expected, norms[i]);
       }

lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java

@@ -23,6 +23,7 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.*;
 import org.apache.lucene.store.Directory;
@@ -224,8 +225,8 @@ public class TestParallelAtomicReader extends LuceneTestCase {
     assertEquals(parallelHits.length, singleHits.length);
     for(int i = 0; i < parallelHits.length; i++) {
       assertEquals(parallelHits[i].score, singleHits[i].score, 0.001f);
-      Document docParallel = parallel.doc(parallelHits[i].doc);
+      StoredDocument docParallel = parallel.doc(parallelHits[i].doc);
-      Document docSingle = single.doc(singleHits[i].doc);
+      StoredDocument docSingle = single.doc(singleHits[i].doc);
       assertEquals(docParallel.get("f1"), docSingle.get("f1"));
       assertEquals(docParallel.get("f2"), docSingle.get("f2"));
       assertEquals(docParallel.get("f3"), docSingle.get("f3"));

lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java

@@ -23,6 +23,7 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.*;
 import org.apache.lucene.store.Directory;
@@ -283,8 +284,8 @@ public class TestParallelCompositeReader extends LuceneTestCase {
     assertEquals(parallelHits.length, singleHits.length);
     for(int i = 0; i < parallelHits.length; i++) {
       assertEquals(parallelHits[i].score, singleHits[i].score, 0.001f);
-      Document docParallel = parallel.doc(parallelHits[i].doc);
+      StoredDocument docParallel = parallel.doc(parallelHits[i].doc);
-      Document docSingle = single.doc(singleHits[i].doc);
+      StoredDocument docSingle = single.doc(singleHits[i].doc);
       assertEquals(docParallel.get("f1"), docSingle.get("f1"));
       assertEquals(docParallel.get("f2"), docSingle.get("f2"));
       assertEquals(docParallel.get("f3"), docSingle.get("f3"));

lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java

@@ -240,7 +240,7 @@ public class TestPostingsOffsets extends LuceneTestCase {
 
     for(int docCount=0;docCount<numDocs;docCount++) {
       Document doc = new Document();
-      doc.add(new IntField("id", docCount, Field.Store.NO));
+      doc.add(new IntField("id", docCount, Field.Store.YES));
       List<Token> tokens = new ArrayList<Token>();
       final int numTokens = atLeast(100);
       //final int numTokens = atLeast(20);

lucene/core/src/test/org/apache/lucene/index/TestRandomStoredFields.java

@@ -29,6 +29,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
@@ -121,7 +122,7 @@ public class TestRandomStoredFields extends LuceneTestCase {
           }
           TopDocs hits = s.search(new TermQuery(new Term("id", testID)), 1);
           assertEquals(1, hits.totalHits);
-          Document doc = r.document(hits.scoreDocs[0].doc);
+          StoredDocument doc = r.document(hits.scoreDocs[0].doc);
           Document docExp = docs.get(testID);
           for(int i=0;i<fieldCount;i++) {
             assertEquals("doc " + testID + ", field f" + fieldCount + " is wrong", docExp.get("f"+i),  doc.get("f"+i));

lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
@@ -96,11 +97,11 @@ public class TestSegmentMerger extends LuceneTestCase {
                                                    DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random()));
     assertTrue(mergedReader != null);
     assertTrue(mergedReader.numDocs() == 2);
-    Document newDoc1 = mergedReader.document(0);
+    StoredDocument newDoc1 = mergedReader.document(0);
     assertTrue(newDoc1 != null);
     //There are 2 unstored fields on the document
     assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
-    Document newDoc2 = mergedReader.document(1);
+    StoredDocument newDoc2 = mergedReader.document(1);
     assertTrue(newDoc2 != null);
     assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
 

lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java

@@ -23,6 +23,7 @@ import java.util.HashSet;
 import java.util.List;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -62,13 +63,13 @@ public class TestSegmentReader extends LuceneTestCase {
   public void testDocument() throws IOException {
     assertTrue(reader.numDocs() == 1);
     assertTrue(reader.maxDoc() >= 1);
-    Document result = reader.document(0);
+    StoredDocument result = reader.document(0);
     assertTrue(result != null);
     //There are 2 unstored fields on the document that are not preserved across writing
     assertTrue(DocHelper.numFields(result) == DocHelper.numFields(testDoc) - DocHelper.unstored.size());
     
-    List<IndexableField> fields = result.getFields();
+    List<StorableField> fields = result.getFields();
-    for (final IndexableField field : fields ) { 
+    for (final StorableField field : fields ) { 
       assertTrue(field != null);
       assertTrue(DocHelper.nameValues.containsKey(field.name()));
     }

lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java

@@ -32,6 +32,7 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -133,8 +134,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
 
   static Term idTerm = new Term("id","");
   IndexingThread[] threads;
-  static Comparator<IndexableField> fieldNameComparator = new Comparator<IndexableField>() {
+  static Comparator<GeneralField> fieldNameComparator = new Comparator<GeneralField>() {
-    public int compare(IndexableField o1, IndexableField o2) {
+    public int compare(GeneralField o1, GeneralField o2) {
       return o1.name().compareTo(o2.name());
     }
   };
@@ -287,7 +288,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
       Bits liveDocs = ((AtomicReader)sub).getLiveDocs();
       System.out.println("  " + ((SegmentReader) sub).getSegmentInfo());
       for(int docID=0;docID<sub.maxDoc();docID++) {
-        Document doc = sub.document(docID);
+        StoredDocument doc = sub.document(docID);
         if (liveDocs == null || liveDocs.get(docID)) {
           System.out.println("    docID=" + docID + " id:" + doc.get("id"));
         } else {
@@ -576,9 +577,9 @@ public class TestStressIndexing2 extends LuceneTestCase {
     }
   }
 
-  public static void verifyEquals(Document d1, Document d2) {
+  public static void verifyEquals(StoredDocument d1, StoredDocument d2) {
-    List<IndexableField> ff1 = d1.getFields();
+    List<StorableField> ff1 = d1.getFields();
-    List<IndexableField> ff2 = d2.getFields();
+    List<StorableField> ff2 = d2.getFields();
 
     Collections.sort(ff1, fieldNameComparator);
     Collections.sort(ff2, fieldNameComparator);
@@ -586,8 +587,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
     assertEquals(ff1 + " : " + ff2, ff1.size(), ff2.size());
 
     for (int i=0; i<ff1.size(); i++) {
-      IndexableField f1 = ff1.get(i);
+      StorableField f1 = ff1.get(i);
-      IndexableField f2 = ff2.get(i);
+      StorableField f2 = ff2.get(i);
       if (f1.binaryValue() != null) {
         assert(f2.binaryValue() != null);
       } else {

lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java

@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
@@ -343,12 +344,12 @@ public class TestStressNRT extends LuceneTestCase {
                 if (results.totalHits != 1) {
                   System.out.println("FAIL: hits id:" + id + " val=" + val);
                   for(ScoreDoc sd : results.scoreDocs) {
-                    final Document doc = r.document(sd.doc);
+                    final StoredDocument doc = r.document(sd.doc);
                     System.out.println("  docID=" + sd.doc + " id:" + doc.get("id") + " foundVal=" + doc.get(field));
                   }
                   fail("id=" + id + " reader=" + r + " totalHits=" + results.totalHits);
                 }
-                Document doc = searcher.doc(results.scoreDocs[0].doc);
+                StoredDocument doc = searcher.doc(results.scoreDocs[0].doc);
                 long foundVal = Long.parseLong(doc.get(field));
                 if (foundVal < Math.abs(val)) {
                   fail("foundVal=" + foundVal + " val=" + val + " id=" + id + " reader=" + r);

lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java

@@ -156,7 +156,7 @@ public class TestTermsEnum extends LuceneTestCase {
 
   private void addDoc(RandomIndexWriter w, Collection<String> terms, Map<BytesRef,Integer> termToID, int id) throws IOException {
     Document doc = new Document();
-    doc.add(new IntField("id", id, Field.Store.NO));
+    doc.add(new IntField("id", id, Field.Store.YES));
     if (VERBOSE) {
       System.out.println("TEST: addDoc id:" + id + " terms=" + terms);
     }

lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java

@@ -18,6 +18,7 @@ package org.apache.lucene.search;
  */
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
@@ -380,7 +381,7 @@ public class TestBooleanMinShouldMatch extends LuceneTestCase {
         DecimalFormat f = new DecimalFormat("0.000000");
 
         for (int i = 0; i < h.length; i++) {
-            Document d = searcher.doc(h[i].doc);
+            StoredDocument d = searcher.doc(h[i].doc);
             float score = h[i].score;
             System.err.println("#" + i + ": " + f.format(score) + " - " +
                                d.get("id") + " - " + d.get("data"));

lucene/core/src/test/org/apache/lucene/search/TestDateSort.java

@@ -25,6 +25,7 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.store.Directory;
@@ -82,7 +83,7 @@ public class TestDateSort extends LuceneTestCase {
     String[] actualOrder = new String[5];
     ScoreDoc[] hits = searcher.search(query, null, 1000, sort).scoreDocs;
     for (int i = 0; i < hits.length; i++) {
-      Document document = searcher.doc(hits[i].doc);
+      StoredDocument document = searcher.doc(hits[i].doc);
       String text = document.get(TEXT_FIELD);
       actualOrder[i] = text;
     }

lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java

@@ -22,6 +22,7 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
@@ -488,7 +489,7 @@ public class TestDisjunctionMaxQuery extends LuceneTestCase {
     DecimalFormat f = new DecimalFormat("0.000000000");
     
     for (int i = 0; i < h.length; i++) {
-      Document d = searcher.doc(h[i].doc);
+      StoredDocument d = searcher.doc(h[i].doc);
       float score = h[i].score;
       System.err
           .println("#" + i + ": " + f.format(score) + " - " + d.get("id"));

lucene/core/src/test/org/apache/lucene/search/TestNRTManager.java

@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexDocument;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -75,7 +76,7 @@ public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase {
   }
 
   @Override
-  protected void updateDocuments(Term id, List<? extends Iterable<? extends IndexableField>> docs) throws Exception {
+  protected void updateDocuments(Term id, List<? extends IndexDocument> docs) throws Exception {
     final long gen = genWriter.updateDocuments(id, docs);
 
     // Randomly verify the update "took":
@@ -99,7 +100,7 @@ public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase {
   }
 
   @Override
-  protected void addDocuments(Term id, List<? extends Iterable<? extends IndexableField>> docs) throws Exception {
+  protected void addDocuments(Term id, List<? extends IndexDocument> docs) throws Exception {
     final long gen = genWriter.addDocuments(docs);
     // Randomly verify the add "took":
     if (random().nextInt(20) == 2) {
@@ -121,7 +122,7 @@ public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase {
   }
 
   @Override
-  protected void addDocument(Term id, Iterable<? extends IndexableField> doc) throws Exception {
+  protected void addDocument(Term id, IndexDocument doc) throws Exception {
     final long gen = genWriter.addDocument(doc);
 
     // Randomly verify the add "took":
@@ -144,7 +145,7 @@ public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase {
   }
 
   @Override
-  protected void updateDocument(Term id, Iterable<? extends IndexableField> doc) throws Exception {
+  protected void updateDocument(Term id, IndexDocument doc) throws Exception {
     final long gen = genWriter.updateDocument(id, doc);
     // Randomly verify the udpate "took":
     if (random().nextInt(20) == 2) {
@@ -373,7 +374,7 @@ public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase {
     }
 
     public void updateDocument(Term term,
-        Iterable<? extends IndexableField> doc, Analyzer analyzer)
+        IndexDocument doc, Analyzer analyzer)
         throws IOException {
       super.updateDocument(term, doc, analyzer);
       try {

lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java

@@ -23,6 +23,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.FloatField;
 import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -173,7 +174,7 @@ public class TestNumericRangeQuery32 extends LuceneTestCase {
       ScoreDoc[] sd = topDocs.scoreDocs;
       assertNotNull(sd);
       assertEquals("Score doc count"+type, count, sd.length );
-      Document doc=searcher.doc(sd[0].doc);
+      StoredDocument doc=searcher.doc(sd[0].doc);
       assertEquals("First doc"+type, 2*distance+startOffset, doc.getField(field).numericValue().intValue());
       doc=searcher.doc(sd[sd.length-1].doc);
       assertEquals("Last doc"+type, (1+count)*distance+startOffset, doc.getField(field).numericValue().intValue());
@@ -227,7 +228,7 @@ public class TestNumericRangeQuery32 extends LuceneTestCase {
     ScoreDoc[] sd = topDocs.scoreDocs;
     assertNotNull(sd);
     assertEquals("Score doc count", count, sd.length );
-    Document doc=searcher.doc(sd[0].doc);
+    StoredDocument doc=searcher.doc(sd[0].doc);
     assertEquals("First doc", startOffset, doc.getField(field).numericValue().intValue());
     doc=searcher.doc(sd[sd.length-1].doc);
     assertEquals("Last doc", (count-1)*distance+startOffset, doc.getField(field).numericValue().intValue());
@@ -267,7 +268,7 @@ public class TestNumericRangeQuery32 extends LuceneTestCase {
     ScoreDoc[] sd = topDocs.scoreDocs;
     assertNotNull(sd);
     assertEquals("Score doc count", noDocs-count, sd.length );
-    Document doc=searcher.doc(sd[0].doc);
+    StoredDocument doc=searcher.doc(sd[0].doc);
     assertEquals("First doc", count*distance+startOffset, doc.getField(field).numericValue().intValue());
     doc=searcher.doc(sd[sd.length-1].doc);
     assertEquals("Last doc", (noDocs-1)*distance+startOffset, doc.getField(field).numericValue().intValue());

lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java

@@ -23,6 +23,7 @@ import org.apache.lucene.document.DoubleField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -182,7 +183,7 @@ public class TestNumericRangeQuery64 extends LuceneTestCase {
       ScoreDoc[] sd = topDocs.scoreDocs;
       assertNotNull(sd);
       assertEquals("Score doc count"+type, count, sd.length );
-      Document doc=searcher.doc(sd[0].doc);
+      StoredDocument doc=searcher.doc(sd[0].doc);
       assertEquals("First doc"+type, 2*distance+startOffset, doc.getField(field).numericValue().longValue() );
       doc=searcher.doc(sd[sd.length-1].doc);
       assertEquals("Last doc"+type, (1+count)*distance+startOffset, doc.getField(field).numericValue().longValue() );
@@ -242,7 +243,7 @@ public class TestNumericRangeQuery64 extends LuceneTestCase {
     ScoreDoc[] sd = topDocs.scoreDocs;
     assertNotNull(sd);
     assertEquals("Score doc count", count, sd.length );
-    Document doc=searcher.doc(sd[0].doc);
+    StoredDocument doc=searcher.doc(sd[0].doc);
     assertEquals("First doc", startOffset, doc.getField(field).numericValue().longValue() );
     doc=searcher.doc(sd[sd.length-1].doc);
     assertEquals("Last doc", (count-1)*distance+startOffset, doc.getField(field).numericValue().longValue() );
@@ -287,7 +288,7 @@ public class TestNumericRangeQuery64 extends LuceneTestCase {
     ScoreDoc[] sd = topDocs.scoreDocs;
     assertNotNull(sd);
     assertEquals("Score doc count", noDocs-count, sd.length );
-    Document doc=searcher.doc(sd[0].doc);
+    StoredDocument doc=searcher.doc(sd[0].doc);
     assertEquals("First doc", count*distance+startOffset, doc.getField(field).numericValue().longValue() );
     doc=searcher.doc(sd[sd.length-1].doc);
     assertEquals("Last doc", (noDocs-1)*distance+startOffset, doc.getField(field).numericValue().longValue() );

lucene/core/src/test/org/apache/lucene/search/TestSort.java

@@ -38,6 +38,7 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.FloatDocValuesField;
 import org.apache.lucene.document.PackedLongDocValuesField;
 import org.apache.lucene.document.SortedBytesDocValuesField;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StraightBytesDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
@@ -50,6 +51,7 @@ import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.StorableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.FieldValueHitQueue.Entry;
@@ -522,9 +524,9 @@ public class TestSort extends LuceneTestCase {
     boolean fail = false;
     final String fieldSuffix = sort.getSort()[0].getField().endsWith("_fixed") ? "_fixed" : "";
     for (int x = 0; x < n; ++x) {
-      Document doc2 = searcher.doc(result[x].doc);
+      StoredDocument doc2 = searcher.doc(result[x].doc);
-      IndexableField[] v = doc2.getFields("tracer" + fieldSuffix);
+      StorableField[] v = doc2.getFields("tracer" + fieldSuffix);
-      IndexableField[] v2 = doc2.getFields("tracer2" + fieldSuffix);
+      StorableField[] v2 = doc2.getFields("tracer2" + fieldSuffix);
       for (int j = 0; j < v.length; ++j) {
         buff.append(v[j] + "(" + v2[j] + ")(" + result[x].doc+")\n");
         if (last != null) {
@@ -1229,8 +1231,8 @@ public class TestSort extends LuceneTestCase {
     StringBuilder buff = new StringBuilder(10);
     int n = result.length;
     for (int i=0; i<n; ++i) {
-      Document doc = searcher.doc(result[i].doc);
+      StoredDocument doc = searcher.doc(result[i].doc);
-      IndexableField[] v = doc.getFields("tracer");
+      StorableField[] v = doc.getFields("tracer");
       for (int j=0; j<v.length; ++j) {
         buff.append (v[j].stringValue());
       }

lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java

@@ -20,6 +20,7 @@ package org.apache.lucene.search.spans;
 import java.io.IOException;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.util.LuceneTestCase;
 
 import org.apache.lucene.analysis.MockAnalyzer;
@@ -160,7 +161,7 @@ public class TestSpansAdvanced extends LuceneTestCase {
       
       int id = topdocs.scoreDocs[i].doc;
       float score = topdocs.scoreDocs[i].score;
-      Document doc = s.doc(id);
+      StoredDocument doc = s.doc(id);
       assertEquals(expectedIds[i], doc.get(FIELD_ID));
       boolean scoreEq = Math.abs(expectedScores[i] - score) < tolerance;
       if (!scoreEq) {

lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java

@@ -21,6 +21,7 @@ import java.io.File;
 import java.io.IOException;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 import org.apache.lucene.analysis.MockAnalyzer;
@@ -85,7 +86,7 @@ public class TestRAMDirectory extends LuceneTestCase {
     
     // search for all documents
     for (int i = 0; i < docsToAdd; i++) {
-      Document doc = searcher.doc(i);
+      StoredDocument doc = searcher.doc(i);
       assertTrue(doc.getField("content") != null);
     }
 

lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java

@@ -27,6 +27,7 @@ import java.util.Date;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryparser.classic.QueryParser;
@@ -178,7 +179,7 @@ public class SearchFiles {
           continue;
         }
 
-        Document doc = searcher.doc(hits[i].doc);
+        StoredDocument doc = searcher.doc(hits[i].doc);
         String path = doc.get("path");
         if (path != null) {
           System.out.println((i+1) + ". " + path);

lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java

@@ -36,6 +36,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -111,7 +112,7 @@ public class FormBasedXmlQueryDemo extends HttpServlet {
       //and package the results and forward to JSP
       if (topDocs != null) {
         ScoreDoc[] sd = topDocs.scoreDocs;
-        Document[] results = new Document[sd.length];
+        StoredDocument[] results = new StoredDocument[sd.length];
         for (int i = 0; i < results.length; i++) {
           results[i] = searcher.doc(sd[i].doc);
           request.setAttribute("results", results);

lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java

@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
@@ -34,6 +35,7 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.StorableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -238,8 +240,8 @@ public abstract class CollationTestBase extends LuceneTestCase {
     StringBuilder buff = new StringBuilder(10);
     int n = result.length;
     for (int i = 0 ; i < n ; ++i) {
-      Document doc = searcher.doc(result[i].doc);
+      StoredDocument doc = searcher.doc(result[i].doc);
-      IndexableField[] v = doc.getFields("tracer");
+      StorableField[] v = doc.getFields("tracer");
       for (int j = 0 ; j < v.length ; ++j) {
         buff.append(v[j].stringValue());
       }

lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java

@@ -29,6 +29,7 @@ import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
@@ -291,6 +292,10 @@ class DocHelper {
     return doc.getFields().size();
   }
 
+  public static int numFields(StoredDocument doc) {
+    return doc.getFields().size();
+  }
+  
   public static Document createDocument(int n, String indexName, int numFields) {
     StringBuilder sb = new StringBuilder();
     FieldType customType = new FieldType(TextField.TYPE_STORED);

lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java

@@ -141,11 +141,11 @@ public class RandomIndexWriter implements Closeable {
    * Adds a Document.
    * @see IndexWriter#addDocument(Iterable)
    */
-  public <T extends IndexableField> void addDocument(final Iterable<T> doc) throws IOException {
+  public <T extends IndexableField> void addDocument(final IndexDocument doc) throws IOException {
     addDocument(doc, w.getAnalyzer());
   }
 
-  public <T extends IndexableField> void addDocument(final Iterable<T> doc, Analyzer a) throws IOException {
+  public <T extends IndexableField> void addDocument(final IndexDocument doc, Analyzer a) throws IOException {
     if (doDocValues && doc instanceof Document) {
       randomPerDocFieldValues((Document) doc);
     }
@@ -154,11 +154,11 @@ public class RandomIndexWriter implements Closeable {
       // (but we need to clone them), and only when
       // getReader, commit, etc. are called, we do an
       // addDocuments?  Would be better testing.
-      w.addDocuments(new Iterable<Iterable<T>>() {
+      w.addDocuments(new Iterable<IndexDocument>() {
 
         @Override
-        public Iterator<Iterable<T>> iterator() {
+        public Iterator<IndexDocument> iterator() {
-          return new Iterator<Iterable<T>>() {
+          return new Iterator<IndexDocument>() {
             boolean done;
             
             @Override
@@ -172,7 +172,7 @@ public class RandomIndexWriter implements Closeable {
             }
 
             @Override
-            public Iterable<T> next() {
+            public IndexDocument next() {
               if (done) {
                 throw new IllegalStateException();
               }
@@ -273,12 +273,12 @@ public class RandomIndexWriter implements Closeable {
     }
   }
   
-  public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+  public void addDocuments(Iterable<? extends IndexDocument> docs) throws IOException {
     w.addDocuments(docs);
     maybeCommit();
   }
 
-  public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+  public void updateDocuments(Term delTerm, Iterable<? extends IndexDocument> docs) throws IOException {
     w.updateDocuments(delTerm, docs);
     maybeCommit();
   }
@@ -287,16 +287,16 @@ public class RandomIndexWriter implements Closeable {
    * Updates a document.
    * @see IndexWriter#updateDocument(Term, Iterable)
    */
-  public <T extends IndexableField> void updateDocument(Term t, final Iterable<T> doc) throws IOException {
+  public <T extends IndexableField> void updateDocument(Term t, final IndexDocument doc) throws IOException {
     if (doDocValues) {
       randomPerDocFieldValues((Document) doc);
     }
     if (r.nextInt(5) == 3) {
-      w.updateDocuments(t, new Iterable<Iterable<T>>() {
+      w.updateDocuments(t, new Iterable<IndexDocument>() {
 
         @Override
-        public Iterator<Iterable<T>> iterator() {
+        public Iterator<IndexDocument> iterator() {
-          return new Iterator<Iterable<T>>() {
+          return new Iterator<IndexDocument>() {
             boolean done;
             
             @Override
@@ -310,7 +310,7 @@ public class RandomIndexWriter implements Closeable {
             }
 
             @Override
-            public Iterable<T> next() {
+            public IndexDocument next() {
               if (done) {
                 throw new IllegalStateException();
               }

lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java

@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredDocument;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
@@ -89,19 +90,19 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
     return in;
   }
 
-  protected void updateDocuments(Term id, List<? extends Iterable<? extends IndexableField>> docs) throws Exception {
+  protected void updateDocuments(Term id, List<? extends IndexDocument> docs) throws Exception {
     writer.updateDocuments(id, docs);
   }
 
-  protected void addDocuments(Term id, List<? extends Iterable<? extends IndexableField>> docs) throws Exception {
+  protected void addDocuments(Term id, List<? extends IndexDocument> docs) throws Exception {
     writer.addDocuments(docs);
   }
 
-  protected void addDocument(Term id, Iterable<? extends IndexableField> doc) throws Exception {
+  protected void addDocument(Term id, IndexDocument doc) throws Exception {
     writer.addDocument(doc);
   }
 
-  protected void updateDocument(Term term, Iterable<? extends IndexableField> doc) throws Exception {
+  protected void updateDocument(Term term, IndexDocument doc) throws Exception {
     writer.updateDocument(term, doc);
   }
 
@@ -464,7 +465,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
         final int inc = Math.max(1, maxDoc/50);
         for(int docID=0;docID<maxDoc;docID += inc) {
           if (liveDocs == null || liveDocs.get(docID)) {
-            final Document doc = reader.document(docID);
+            final StoredDocument doc = reader.document(docID);
             sum += doc.getFields().size();
           }
         }
@@ -564,7 +565,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
               startDocID = docID;
             }
             lastDocID = docID;
-            final Document doc = s.doc(docID);
+            final StoredDocument doc = s.doc(docID);
             assertEquals(subDocs.packID, doc.get("packID"));
           }
 

lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java

@@ -792,7 +792,7 @@ public class _TestUtil {
   // TODO: is there a pre-existing way to do this!!!
   public static Document cloneDocument(Document doc1) {
     final Document doc2 = new Document();
-    for(IndexableField f : doc1) {
+    for(IndexableField f : doc1.getFields()) {
       final Field field1 = (Field) f;
       final Field field2;
       final DocValues.Type dvType = field1.fieldType().docValueType();