SOLR-52 lazyfields patch implemented.

I am going to commit this with solrconfig defaulting to disabling lazyfields and use it in my 
application.  When I'm satisfied with the behaviour, I'll close SOLR-52 and perhaps enable 
lazyfields as the default option.



git-svn-id: https://svn.apache.org/repos/asf/incubator/solr/trunk@479793 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -88,6 +88,8 @@ Changes in runtime behavior
     (hossman, SOLR-25)
  8. Document update handling locking is much sparser, allowing performance gains
     through multiple threads.  Large commits also might be faster (klaas, SOLR-65)
+ 9. Lazy field loading can be enabled via a solrconfig directive.  This will be faster when
+    not all stored fields are needed from a document (klaas, SOLR-52)   
 
 Optimizations 
  1. getDocListAndSet can now generate both a DocList and a DocSet from a 

example/solr/conf/solrconfig.xml

@@ -132,6 +132,10 @@
       initialSize="512"
       autowarmCount="0"/>
 
+    <!-- If true, stored fields that are not requested will be loaded lazily.
+    -->
+    <enableLazyFieldLoading>false</enableLazyFieldLoading>
+
     <!-- Example of a generic cache.  These caches may be accessed by name
          through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
          The purpose is to enable easy caching of user/application level data.

src/java/org/apache/solr/request/DisMaxRequestHandler.java

@@ -347,6 +347,9 @@ public class DisMaxRequestHandler
                                        flags);
       }
       rsp.add("search-results",results.docList);
+      // pre-fetch returned documents
+      U.optimizePreFetchDocs(results.docList, query, req, rsp);
+
       
       if (null != facetInfo) rsp.add("facet_counts", facetInfo);
 

src/java/org/apache/solr/request/JSONResponseWriter.java

@@ -382,7 +382,7 @@ class JSONWriter extends TextResponseWriter {
     DocIterator iterator = ids.iterator();
     for (int i=0; i<sz; i++) {
       int id = iterator.nextDoc();
-      Document doc = searcher.doc(id);
+      Document doc = searcher.doc(id, fields);
 
       if (first) {
         first=false;

src/java/org/apache/solr/request/StandardRequestHandler.java

@@ -136,6 +136,9 @@ public class StandardRequestHandler implements SolrRequestHandler, SolrInfoMBean
                                        flags);
       }
 
+      // pre-fetch returned documents
+      U.optimizePreFetchDocs(results.docList, query, req, rsp);
+      
       rsp.add(null,results.docList);
 
       if (null != facetInfo) rsp.add("facet_counts", facetInfo);
@@ -154,7 +157,7 @@ public class StandardRequestHandler implements SolrRequestHandler, SolrInfoMBean
           rsp.add("debug", dbg);
         }
       } catch (Exception e) {
-        SolrException.logOnce(SolrCore.log, "Exception durring debug", e);
+        SolrException.logOnce(SolrCore.log, "Exception during debug", e);
         rsp.add("exception_during_debug", SolrException.toStr(e));
       }
 

src/java/org/apache/solr/request/XMLWriter.java

@@ -30,7 +30,7 @@ import java.io.Writer;
 import java.io.IOException;
 import java.util.*;
 
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.Document;
 /**
  * @author yonik
@@ -215,7 +215,7 @@ final public class XMLWriter {
 
   private static final Comparator fieldnameComparator = new Comparator() {
     public int compare(Object o, Object o1) {
-      Field f1 = (Field)o; Field f2 = (Field)o1;
+      Fieldable f1 = (Fieldable)o; Fieldable f2 = (Fieldable)o1;
       int cmp = f1.name().compareTo(f2.name());
       return cmp;
       // note - the sort is stable, so this should not have affected the ordering
@@ -238,13 +238,12 @@ final public class XMLWriter {
     // an array.  The fastest way to detect multiple fields
     // with the same name is to sort them first.
 
-    Enumeration ee = doc.fields();
 
     // using global tlst here, so we shouldn't call any other
     // function that uses it until we are done.
     tlst.clear();
-    while (ee.hasMoreElements()) {
+    for (Object obj : doc.getFields()) {
-      Field ff = (Field) ee.nextElement();
+      Fieldable ff = (Fieldable)obj;
       // skip this field if it is not a field to be returned.
       if (returnFields!=null && !returnFields.contains(ff.name())) {
         continue;
@@ -256,12 +255,12 @@ final public class XMLWriter {
     int sz = tlst.size();
     int fidx1 = 0, fidx2 = 0;
     while (fidx1 < sz) {
-      Field f1 = (Field)tlst.get(fidx1);
+      Fieldable f1 = (Fieldable)tlst.get(fidx1);
       String fname = f1.name();
 
       // find the end of fields with this name
       fidx2 = fidx1+1;
-      while (fidx2 < sz && fname.equals(((Field)tlst.get(fidx2)).name()) ) {
+      while (fidx2 < sz && fname.equals(((Fieldable)tlst.get(fidx2)).name()) ) {
         fidx2++;
       }
 
@@ -297,7 +296,7 @@ final public class XMLWriter {
             indent();
             cnt=0;
           }
-          sf.write(this, null, (Field)tlst.get(i));
+          sf.write(this, null, (Fieldable)tlst.get(i));
         }
         decLevel();
         // if (doIndent) indent();
@@ -343,7 +342,7 @@ final public class XMLWriter {
     DocIterator iterator = ids.iterator();
     for (int i=0; i<sz; i++) {
       int id = iterator.nextDoc();
-      Document doc = searcher.doc(id);
+      Document doc = searcher.doc(id, fields);
       writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
     }
     decLevel();

src/java/org/apache/solr/schema/BCDIntField.java

@@ -19,7 +19,6 @@ package org.apache.solr.schema;
 
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.util.BCDUtils;
 import org.apache.solr.request.XMLWriter;
@@ -47,7 +46,7 @@ public class BCDIntField extends FieldType {
     return BCDUtils.base10toBase10kSortableInt(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -55,7 +54,7 @@ public class BCDIntField extends FieldType {
     return BCDUtils.base10kSortableIntToBase10(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeInt(name,toExternal(f));
   }
 

src/java/org/apache/solr/schema/BCDLongField.java

@@ -18,7 +18,7 @@
 package org.apache.solr.schema;
 
 import org.apache.solr.request.XMLWriter;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 
 import java.io.IOException;
 /**
@@ -26,7 +26,7 @@ import java.io.IOException;
  * @version $Id$
  */
 public class BCDLongField extends BCDIntField {
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeLong(name,toExternal(f));
   }
 }

src/java/org/apache/solr/schema/BCDStrField.java

@@ -18,7 +18,7 @@
 package org.apache.solr.schema;
 
 import org.apache.solr.request.XMLWriter;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 
 import java.io.IOException;
 /**
@@ -26,7 +26,7 @@ import java.io.IOException;
  * @version $Id$
  */
 public class BCDStrField extends BCDIntField {
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeStr(name,toExternal(f));
   }
 }

src/java/org/apache/solr/schema/BoolField.java

@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -86,7 +85,7 @@ public class BoolField extends FieldType {
     return (ch=='1' || ch=='t' || ch=='T') ? "T" : "F";
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -95,7 +94,7 @@ public class BoolField extends FieldType {
     return ch=='T' ? "true" : "false";
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeBool(name, f.stringValue().charAt(0) =='T');
   }
 

src/java/org/apache/solr/schema/CompressableField.java

@@ -18,6 +18,7 @@
 package org.apache.solr.schema;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 
 import org.apache.solr.request.*;
 

src/java/org/apache/solr/schema/DateField.java

@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.solr.core.SolrException;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
@@ -117,7 +116,7 @@ public class DateField extends FieldType {
     return indexedForm + 'Z';
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -129,7 +128,7 @@ public class DateField extends FieldType {
     return new OrdFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeDate(name, toExternal(f));
   }
 

src/java/org/apache/solr/schema/DoubleField.java

@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FloatFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -47,7 +46,7 @@ public class DoubleField extends FieldType {
     return new FloatFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeDouble(name, f.stringValue());
   }
 

src/java/org/apache/solr/schema/FieldType.java

@@ -225,7 +225,7 @@ public abstract class FieldType extends FieldProperties {
    */
   public String toExternal(Fieldable f) {
     // currently used in writing XML of the search result (but perhaps
-    // a more efficient toXML(Field f, Writer w) should be used
+    // a more efficient toXML(Fieldable f, Writer w) should be used
     // in the future.
     return f.stringValue();
   }
@@ -357,7 +357,7 @@ public abstract class FieldType extends FieldProperties {
   /**
    * Renders the specified field as XML
    */
-  public abstract void write(XMLWriter xmlWriter, String name, Field f) throws IOException;
+  public abstract void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException;
 
   /**
    * calls back to TextResponseWriter to write the field value

src/java/org/apache/solr/schema/FloatField.java

@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FloatFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -44,7 +43,7 @@ public class FloatField extends FieldType {
     return new FloatFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeFloat(name, f.stringValue());
   }
 

src/java/org/apache/solr/schema/IndexSchema.java

@@ -20,6 +20,7 @@ package org.apache.solr.schema;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.search.DefaultSimilarity;
 import org.apache.lucene.search.Similarity;
 import org.apache.solr.core.SolrException;
@@ -170,8 +171,8 @@ public final class IndexSchema {
    * @return null if this schema has no unique key field
    * @see #printableUniqueKey
    */
-  public Field getUniqueKeyField(org.apache.lucene.document.Document doc) {
+  public Fieldable getUniqueKeyField(org.apache.lucene.document.Document doc) {
-    return doc.getField(uniqueKeyFieldName);  // this should return null if name is null
+    return doc.getFieldable(uniqueKeyFieldName);  // this should return null if name is null
   }
 
   /**
@@ -180,7 +181,7 @@ public final class IndexSchema {
    * @return null if this schema has no unique key field
    */
   public String printableUniqueKey(org.apache.lucene.document.Document doc) {
-     Field f = doc.getField(uniqueKeyFieldName);
+     Fieldable f = doc.getFieldable(uniqueKeyFieldName);
      return f==null ? null : uniqueKeyFieldType.toExternal(f);
   }
 

src/java/org/apache/solr/schema/IntField.java

@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.IntFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -44,7 +43,7 @@ public class IntField extends FieldType {
     return new IntFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeInt(name, f.stringValue());
   }
 

src/java/org/apache/solr/schema/LongField.java

@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.IntFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -49,7 +48,7 @@ public class LongField extends FieldType {
   }
 
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeLong(name, f.stringValue());
   }
 

src/java/org/apache/solr/schema/SchemaField.java

@@ -93,7 +93,7 @@ public final class SchemaField extends FieldProperties {
             + "}";
   }
 
-  public void write(XMLWriter writer, String name, Field val) throws IOException {
+  public void write(XMLWriter writer, String name, Fieldable val) throws IOException {
     // name is passed in because it may be null if name should not be used.
     type.write(writer,name,val);
   }

src/java/org/apache/solr/schema/SortableDoubleField.java

@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -51,7 +50,7 @@ public class SortableDoubleField extends FieldType {
     return NumberUtils.double2sortableStr(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -59,7 +58,7 @@ public class SortableDoubleField extends FieldType {
     return NumberUtils.SortableStr2doubleStr(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     xmlWriter.writeDouble(name, NumberUtils.SortableStr2double(sval));
   }

src/java/org/apache/solr/schema/SortableFloatField.java

@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -51,7 +50,7 @@ public class SortableFloatField extends FieldType {
     return NumberUtils.float2sortableStr(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -59,7 +58,7 @@ public class SortableFloatField extends FieldType {
     return NumberUtils.SortableStr2floatStr(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     xmlWriter.writeFloat(name, NumberUtils.SortableStr2float(sval));
   }

src/java/org/apache/solr/schema/SortableIntField.java

@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -54,7 +53,7 @@ public class SortableIntField extends FieldType {
     return NumberUtils.int2sortableStr(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -62,7 +61,7 @@ public class SortableIntField extends FieldType {
     return NumberUtils.SortableStr2int(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     // since writeInt an int instead of a String since that may be more efficient
     // in the future (saves the construction of one String)

src/java/org/apache/solr/schema/SortableLongField.java

@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -55,11 +54,11 @@ public class SortableLongField extends FieldType {
     return NumberUtils.SortableStr2long(indexedForm);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     xmlWriter.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length()));
   }

src/java/org/apache/solr/schema/StrField.java

@@ -18,7 +18,6 @@
 package org.apache.solr.schema;
 
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -38,7 +37,7 @@ public class StrField extends CompressableField {
     return getStringSort(field,reverse);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeStr(name, f.stringValue());
   }
 

src/java/org/apache/solr/schema/TextField.java

@@ -18,7 +18,6 @@
 package org.apache.solr.schema;
 
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -41,7 +40,7 @@ public class TextField extends CompressableField {
     return new SortField(field.name,SortField.STRING, reverse);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeStr(name, f.stringValue());
   }
 

src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -17,7 +17,7 @@
 
 package org.apache.solr.search;
 
-import org.apache.lucene.document.Document;
+import org.apache.lucene.document.*;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermDocs;
@@ -29,6 +29,7 @@ import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrInfoMBean;
 import org.apache.solr.core.SolrInfoRegistry;
 import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
 import org.apache.solr.util.NamedList;
 import org.apache.solr.util.OpenBitSet;
 
@@ -299,22 +300,90 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
     return searcher.docFreq(term);
   }
 
+  /* ********************** Document retrieval *************************/
+   
+  /* Future optimizations (yonik)
+   *
+   * If no cache is present:
+   *   - use NO_LOAD instead of LAZY_LOAD
+   *   - use LOAD_AND_BREAK if a single field is begin retrieved
+   */
+
+  /**
+   * FieldSelector which loads the specified fields, and load all other
+   * field lazily.
+   */
+  class SetNonLazyFieldSelector implements FieldSelector {
+    private Set<String> fieldsToLoad;
+    SetNonLazyFieldSelector(Set<String> toLoad) {
+      fieldsToLoad = toLoad;
+    }
+    public FieldSelectorResult accept(String fieldName) { 
+      if(fieldsToLoad.contains(fieldName))
+        return FieldSelectorResult.LOAD; 
+      else
+        return FieldSelectorResult.LAZY_LOAD;
+    }
+  }
+
+  /* solrconfig lazyfields setting */
+  public static final boolean enableLazyFieldLoading = SolrConfig.config.getBool("query/enableLazyFieldLoading", false);
+
+  /**
+   * Retrieve the {@link Document} instance corresponding to the document id.
+   */
   public Document doc(int i) throws IOException {
+    return doc(i, null);
+  }
+  /**
+   * Retrieve the {@link Document} instance corresponding to the document id.
+   *
+   * Note: The document will have all fields accessable, but if a field
+   * filter is provided, only the provided fields will be loaded (the 
+   * remainder will be available lazily).
+   */
+  public Document doc(int i, Set<String> fields) throws IOException {
+    
     Document d;
     if (documentCache != null) {
       d = (Document)documentCache.get(i);
       if (d!=null) return d;
     }
 
-    d = searcher.doc(i);
+    if(!enableLazyFieldLoading || fields == null) {
+      d = searcher.getIndexReader().document(i);
+    } else {
+      d = searcher.getIndexReader().document(i, 
+             new SetNonLazyFieldSelector(fields));
+    }
 
     if (documentCache != null) {
-      documentCache.put(i,d);
+      documentCache.put(i, d);
     }
 
     return d;
   }
 
+  /**
+   * Takes a list of docs (the doc ids actually), and reads them into an array 
+   * of Documents.
+   */
+  public void readDocs(Document[] docs, DocList ids) throws IOException {
+    readDocs(docs, ids, null);
+  }
+  /**
+   * Takes a list of docs (the doc ids actually) and a set of fields to load,
+   * and reads them into an array of Documents.
+   */
+  public void readDocs(Document[] docs, DocList ids, Set<String> fields) throws IOException {
+    DocIterator iter = ids.iterator();
+    for (int i=0; i<docs.length; i++) {
+      docs[i] = doc(iter.nextDoc(), fields);
+    }
+  }
+
+  /* ********************** end document retrieval *************************/
+
   public int maxDoc() throws IOException {
     return searcher.maxDoc();
   }
@@ -1236,17 +1305,6 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
      return docs;
   }
 
-  /**
-   * Takes a list of docs (the doc ids actually), and reads them into an array 
-   * of Documents.
-   */
-  public void readDocs(Document[] docs, DocList ids) throws IOException {
-    DocIterator iter = ids.iterator();
-    for (int i=0; i<docs.length; i++) {
-      docs[i] = doc(iter.nextDoc());
-    }
-  }
-
 
 
   /**

src/java/org/apache/solr/util/HighlightingUtils.java

@@ -20,12 +20,15 @@ import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.solr.request.*;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
 import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.schema.SchemaField;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
@@ -215,13 +218,24 @@ public class HighlightingUtils
       SolrIndexSearcher searcher = req.getSearcher();
       NamedList fragments = new NamedList();
       String[] fieldNames = getHighlightFields(query, req, defaultFields);
+      Document[] readDocs = new Document[docs.size()];
+      {
+        // pre-fetch documents using the Searcher's doc cache
+        Set<String> fset = new HashSet<String>();
+        for(String f : fieldNames) { fset.add(f); }
+        // fetch unique key if one exists.
+        SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
+        if(null != keyField)
+          fset.add(keyField.getName());  
+        searcher.readDocs(readDocs, docs, fset);
+      }
 
+      // Highlight each document
       DocIterator iterator = docs.iterator();
       for (int i = 0; i < docs.size(); i++)
       {
          int docId = iterator.nextDoc();
-         // use the Searcher's doc cache
+         Document doc = readDocs[i];
-         Document doc = searcher.doc(docId);
          NamedList docSummaries = new NamedList();
          for (String fieldName : fieldNames)
          {

src/java/org/apache/solr/util/SolrPluginUtils.java

@@ -35,6 +35,7 @@ import org.apache.solr.request.SolrQueryResponse;
 import org.apache.solr.request.DefaultSolrParams;
 import org.apache.solr.request.AppendedSolrParams;
 import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.*;
 
 import java.io.IOException;
@@ -205,6 +206,57 @@ public class SolrPluginUtils {
     return flags;
   }
 
+  /**
+   * Pre-fetch documents into the index searcher's document cache.
+   *
+   * This is an entirely optional step which you might want to perform for
+   * the following reasons:
+   *
+   * <ul>
+   *     <li>Locates the document-retrieval costs in one spot, which helps
+   *     detailed performance measurement</li>
+   *   
+   *     <li>Determines a priori what fields will be needed to be fetched by
+   *     various subtasks, like response writing and highlighting.  This
+   *     minimizes the chance that many needed fields will be loaded lazily.
+   *     (it is more efficient to load all the field we require normally).</li>
+   * </ul>
+   *
+   * If lazy field loading is disabled, this method does nothing.
+   */
+  public static void optimizePreFetchDocs(DocList docs,
+                                          Query query,
+                                          SolrQueryRequest req,
+                                          SolrQueryResponse res) throws IOException {
+    SolrIndexSearcher searcher = req.getSearcher();
+    if(!searcher.enableLazyFieldLoading) {
+      // nothing to do
+      return;
+    }
+
+    Set<String> fieldFilter = null;
+    Set<String> returnFields = res.getReturnFields();
+    if(returnFields != null) {
+      // copy return fields list
+      fieldFilter = new HashSet<String>(returnFields);
+      // add highlight fields
+      if(HighlightingUtils.isHighlightingEnabled(req)) {
+        for(String field: HighlightingUtils.getHighlightFields(query, req, null)) 
+          fieldFilter.add(field);        
+      }
+      // fetch unique key if one exists.
+      SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
+      if(null != keyField)
+          fieldFilter.add(keyField.getName());  
+    }
+
+    // get documents
+    DocIterator iter = docs.iterator();
+    for (int i=0; i<docs.size(); i++) {
+      searcher.doc(iter.nextDoc(), fieldFilter);
+    }
+  }
+
   /**
    * <p>
    * Returns a NamedList containing many "standard" pieces of debugging
@@ -794,6 +846,7 @@ public class SolrPluginUtils {
     }
             
   }
+
 }
 
 /** 

src/test/org/apache/solr/BasicFunctionalityTest.java

@@ -17,9 +17,10 @@
 
 package org.apache.solr;
 
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.*;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.solr.core.SolrCore;
 import org.apache.solr.search.*;
 import org.apache.solr.request.*;
 import org.apache.solr.util.*;
@@ -568,6 +569,48 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase {
     
   }
 
+  public void testNotLazyField() throws IOException {
+    for(int i = 0; i < 10; i++) {
+      assertU(adoc("id", new Integer(i).toString(), 
+                   "title", "keyword",
+                   "test_hlt", mkstr(20000)));
+    }
+    assertU(commit());
+    SolrCore core = h.getCore();
+   
+    SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title,test_hlt");
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    core.execute(req, rsp);
+
+    DocList dl = (DocList) rsp.getValues().get(null);
+    org.apache.lucene.document.Document d = req.getSearcher().doc(dl.iterator().nextDoc());
+    // ensure field is not lazy
+    assertTrue( d.getFieldable("test_hlt") instanceof Field );
+    assertTrue( d.getFieldable("title") instanceof Field );
+  }
+
+  public void testLazyField() throws IOException {
+    for(int i = 0; i < 10; i++) {
+      assertU(adoc("id", new Integer(i).toString(), 
+                   "title", "keyword",
+                   "test_hlt", mkstr(20000)));
+    }
+    assertU(commit());
+    SolrCore core = h.getCore();
+    
+    SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title");
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    core.execute(req, rsp);
+
+    DocList dl = (DocList) rsp.getValues().get(null);
+    DocIterator di = dl.iterator();    
+    org.apache.lucene.document.Document d = req.getSearcher().doc(di.nextDoc());
+    // ensure field is lazy
+    assertTrue( !( d.getFieldable("test_hlt") instanceof Field ) );
+    assertTrue( d.getFieldable("title") instanceof Field );
+  } 
+            
+
   /** @see org.apache.solr.util.DateMathParserTest */
   public void testDateMath() {
 

src/test/test-files/solr/conf/solrconfig.xml

@@ -106,7 +106,12 @@
       initialSize="512"
       autowarmCount="0"/>
 
+    <!-- If true, stored fields that are not requested will be loaded lazily.
+    -->
+    <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
     <!--
+
     <cache name="myUserCache"
       class="solr.search.LRUCache"
       size="4096"