From c982fdedd6958a138d44d72d8a4d5e02e5f74443 Mon Sep 17 00:00:00 2001
From: Mike Klaas <klaas@apache.org>
Date: Mon, 27 Nov 2006 22:40:21 +0000
Subject: [PATCH] SOLR-52 lazyfields patch implemented.

I am going to commit this with solrconfig defaulting to disabling lazyfields and use it in my
application.  When I'm satisfied with the behaviour, I'll close SOLR-52 and perhaps enable
lazyfields as the default option.



git-svn-id: https://svn.apache.org/repos/asf/incubator/solr/trunk@479793 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES.txt                                   |  2 +
 example/solr/conf/solrconfig.xml              |  4 +
 .../solr/request/DisMaxRequestHandler.java    |  3 +
 .../solr/request/JSONResponseWriter.java      |  2 +-
 .../solr/request/StandardRequestHandler.java  |  5 +-
 .../org/apache/solr/request/XMLWriter.java    | 17 ++--
 .../org/apache/solr/schema/BCDIntField.java   |  5 +-
 .../org/apache/solr/schema/BCDLongField.java  |  4 +-
 .../org/apache/solr/schema/BCDStrField.java   |  4 +-
 .../org/apache/solr/schema/BoolField.java     |  5 +-
 .../apache/solr/schema/CompressableField.java |  1 +
 .../org/apache/solr/schema/DateField.java     |  5 +-
 .../org/apache/solr/schema/DoubleField.java   |  3 +-
 .../org/apache/solr/schema/FieldType.java     |  4 +-
 .../org/apache/solr/schema/FloatField.java    |  3 +-
 .../org/apache/solr/schema/IndexSchema.java   |  7 +-
 src/java/org/apache/solr/schema/IntField.java |  3 +-
 .../org/apache/solr/schema/LongField.java     |  3 +-
 .../org/apache/solr/schema/SchemaField.java   |  2 +-
 .../solr/schema/SortableDoubleField.java      |  5 +-
 .../solr/schema/SortableFloatField.java       |  5 +-
 .../apache/solr/schema/SortableIntField.java  |  5 +-
 .../apache/solr/schema/SortableLongField.java |  5 +-
 src/java/org/apache/solr/schema/StrField.java |  3 +-
 .../org/apache/solr/schema/TextField.java     |  3 +-
 .../apache/solr/search/SolrIndexSearcher.java | 86 ++++++++++++++++---
 .../apache/solr/util/HighlightingUtils.java   | 18 +++-
 .../org/apache/solr/util/SolrPluginUtils.java | 53 ++++++++++++
 .../apache/solr/BasicFunctionalityTest.java   | 45 +++++++++-
 src/test/test-files/solr/conf/solrconfig.xml  |  5 ++
 30 files changed, 244 insertions(+), 71 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 1231b0753d7..ea4ba438fbb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -88,6 +88,8 @@ Changes in runtime behavior
     (hossman, SOLR-25)
  8. Document update handling locking is much sparser, allowing performance gains
     through multiple threads.  Large commits also might be faster (klaas, SOLR-65)
+ 9. Lazy field loading can be enabled via a solrconfig directive.  This will be faster when
+    not all stored fields are needed from a document (klaas, SOLR-52)   
 
 Optimizations 
  1. getDocListAndSet can now generate both a DocList and a DocSet from a 
diff --git a/example/solr/conf/solrconfig.xml b/example/solr/conf/solrconfig.xml
index 6be4a9c363a..34844fdbae6 100755
--- a/example/solr/conf/solrconfig.xml
+++ b/example/solr/conf/solrconfig.xml
@@ -132,6 +132,10 @@
       initialSize="512"
       autowarmCount="0"/>
 
+    <!-- If true, stored fields that are not requested will be loaded lazily.
+    -->
+    <enableLazyFieldLoading>false</enableLazyFieldLoading>
+
     <!-- Example of a generic cache.  These caches may be accessed by name
          through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
          The purpose is to enable easy caching of user/application level data.
diff --git a/src/java/org/apache/solr/request/DisMaxRequestHandler.java b/src/java/org/apache/solr/request/DisMaxRequestHandler.java
index 236e1f0ce20..cb1afb16cd6 100644
--- a/src/java/org/apache/solr/request/DisMaxRequestHandler.java
+++ b/src/java/org/apache/solr/request/DisMaxRequestHandler.java
@@ -347,6 +347,9 @@ public class DisMaxRequestHandler
                                        flags);
       }
       rsp.add("search-results",results.docList);
+      // pre-fetch returned documents
+      U.optimizePreFetchDocs(results.docList, query, req, rsp);
+
       
       if (null != facetInfo) rsp.add("facet_counts", facetInfo);
 
diff --git a/src/java/org/apache/solr/request/JSONResponseWriter.java b/src/java/org/apache/solr/request/JSONResponseWriter.java
index 7f75b81d78b..b02fb18161b 100644
--- a/src/java/org/apache/solr/request/JSONResponseWriter.java
+++ b/src/java/org/apache/solr/request/JSONResponseWriter.java
@@ -382,7 +382,7 @@ class JSONWriter extends TextResponseWriter {
     DocIterator iterator = ids.iterator();
     for (int i=0; i<sz; i++) {
       int id = iterator.nextDoc();
-      Document doc = searcher.doc(id);
+      Document doc = searcher.doc(id, fields);
 
       if (first) {
         first=false;
diff --git a/src/java/org/apache/solr/request/StandardRequestHandler.java b/src/java/org/apache/solr/request/StandardRequestHandler.java
index b331c940fd7..25c836a258f 100644
--- a/src/java/org/apache/solr/request/StandardRequestHandler.java
+++ b/src/java/org/apache/solr/request/StandardRequestHandler.java
@@ -135,6 +135,9 @@ public class StandardRequestHandler implements SolrRequestHandler, SolrInfoMBean
                                        p.getInt(START,0), p.getInt(ROWS,10),
                                        flags);
       }
+
+      // pre-fetch returned documents
+      U.optimizePreFetchDocs(results.docList, query, req, rsp);
       
       rsp.add(null,results.docList);
 
@@ -154,7 +157,7 @@ public class StandardRequestHandler implements SolrRequestHandler, SolrInfoMBean
           rsp.add("debug", dbg);
         }
       } catch (Exception e) {
-        SolrException.logOnce(SolrCore.log, "Exception durring debug", e);
+        SolrException.logOnce(SolrCore.log, "Exception during debug", e);
         rsp.add("exception_during_debug", SolrException.toStr(e));
       }
 
diff --git a/src/java/org/apache/solr/request/XMLWriter.java b/src/java/org/apache/solr/request/XMLWriter.java
index 58c3064bf74..9f44b023298 100644
--- a/src/java/org/apache/solr/request/XMLWriter.java
+++ b/src/java/org/apache/solr/request/XMLWriter.java
@@ -30,7 +30,7 @@ import java.io.Writer;
 import java.io.IOException;
 import java.util.*;
 
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.Document;
 /**
  * @author yonik
@@ -215,7 +215,7 @@ final public class XMLWriter {
 
   private static final Comparator fieldnameComparator = new Comparator() {
     public int compare(Object o, Object o1) {
-      Field f1 = (Field)o; Field f2 = (Field)o1;
+      Fieldable f1 = (Fieldable)o; Fieldable f2 = (Fieldable)o1;
       int cmp = f1.name().compareTo(f2.name());
       return cmp;
       // note - the sort is stable, so this should not have affected the ordering
@@ -238,13 +238,12 @@ final public class XMLWriter {
     // an array.  The fastest way to detect multiple fields
     // with the same name is to sort them first.
 
-    Enumeration ee = doc.fields();
 
     // using global tlst here, so we shouldn't call any other
     // function that uses it until we are done.
     tlst.clear();
-    while (ee.hasMoreElements()) {
-      Field ff = (Field) ee.nextElement();
+    for (Object obj : doc.getFields()) {
+      Fieldable ff = (Fieldable)obj;
       // skip this field if it is not a field to be returned.
       if (returnFields!=null && !returnFields.contains(ff.name())) {
         continue;
@@ -256,12 +255,12 @@ final public class XMLWriter {
     int sz = tlst.size();
     int fidx1 = 0, fidx2 = 0;
     while (fidx1 < sz) {
-      Field f1 = (Field)tlst.get(fidx1);
+      Fieldable f1 = (Fieldable)tlst.get(fidx1);
       String fname = f1.name();
 
       // find the end of fields with this name
       fidx2 = fidx1+1;
-      while (fidx2 < sz && fname.equals(((Field)tlst.get(fidx2)).name()) ) {
+      while (fidx2 < sz && fname.equals(((Fieldable)tlst.get(fidx2)).name()) ) {
         fidx2++;
       }
 
@@ -297,7 +296,7 @@ final public class XMLWriter {
             indent();
             cnt=0;
           }
-          sf.write(this, null, (Field)tlst.get(i));
+          sf.write(this, null, (Fieldable)tlst.get(i));
         }
         decLevel();
         // if (doIndent) indent();
@@ -343,7 +342,7 @@ final public class XMLWriter {
     DocIterator iterator = ids.iterator();
     for (int i=0; i<sz; i++) {
       int id = iterator.nextDoc();
-      Document doc = searcher.doc(id);
+      Document doc = searcher.doc(id, fields);
       writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
     }
     decLevel();
diff --git a/src/java/org/apache/solr/schema/BCDIntField.java b/src/java/org/apache/solr/schema/BCDIntField.java
index 6e4f057538e..181df07cdd9 100644
--- a/src/java/org/apache/solr/schema/BCDIntField.java
+++ b/src/java/org/apache/solr/schema/BCDIntField.java
@@ -19,7 +19,6 @@ package org.apache.solr.schema;
 
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.util.BCDUtils;
 import org.apache.solr.request.XMLWriter;
@@ -47,7 +46,7 @@ public class BCDIntField extends FieldType {
     return BCDUtils.base10toBase10kSortableInt(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -55,7 +54,7 @@ public class BCDIntField extends FieldType {
     return BCDUtils.base10kSortableIntToBase10(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeInt(name,toExternal(f));
   }
 
diff --git a/src/java/org/apache/solr/schema/BCDLongField.java b/src/java/org/apache/solr/schema/BCDLongField.java
index cb206c18fcb..398d39b6a2f 100644
--- a/src/java/org/apache/solr/schema/BCDLongField.java
+++ b/src/java/org/apache/solr/schema/BCDLongField.java
@@ -18,7 +18,7 @@
 package org.apache.solr.schema;
 
 import org.apache.solr.request.XMLWriter;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 
 import java.io.IOException;
 /**
@@ -26,7 +26,7 @@ import java.io.IOException;
  * @version $Id$
  */
 public class BCDLongField extends BCDIntField {
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeLong(name,toExternal(f));
   }
 }
diff --git a/src/java/org/apache/solr/schema/BCDStrField.java b/src/java/org/apache/solr/schema/BCDStrField.java
index 3073dce5cc2..4b211e3221c 100644
--- a/src/java/org/apache/solr/schema/BCDStrField.java
+++ b/src/java/org/apache/solr/schema/BCDStrField.java
@@ -18,7 +18,7 @@
 package org.apache.solr.schema;
 
 import org.apache.solr.request.XMLWriter;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 
 import java.io.IOException;
 /**
@@ -26,7 +26,7 @@ import java.io.IOException;
  * @version $Id$
  */
 public class BCDStrField extends BCDIntField {
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeStr(name,toExternal(f));
   }
 }
diff --git a/src/java/org/apache/solr/schema/BoolField.java b/src/java/org/apache/solr/schema/BoolField.java
index 6738aa3edfe..e2aa1f602e6 100644
--- a/src/java/org/apache/solr/schema/BoolField.java
+++ b/src/java/org/apache/solr/schema/BoolField.java
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -86,7 +85,7 @@ public class BoolField extends FieldType {
     return (ch=='1' || ch=='t' || ch=='T') ? "T" : "F";
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -95,7 +94,7 @@ public class BoolField extends FieldType {
     return ch=='T' ? "true" : "false";
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeBool(name, f.stringValue().charAt(0) =='T');
   }
 
diff --git a/src/java/org/apache/solr/schema/CompressableField.java b/src/java/org/apache/solr/schema/CompressableField.java
index 28896c9a189..43c1d669dde 100644
--- a/src/java/org/apache/solr/schema/CompressableField.java
+++ b/src/java/org/apache/solr/schema/CompressableField.java
@@ -18,6 +18,7 @@
 package org.apache.solr.schema;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 
 import org.apache.solr.request.*;
 
diff --git a/src/java/org/apache/solr/schema/DateField.java b/src/java/org/apache/solr/schema/DateField.java
index 8174451372f..faafc7abb81 100644
--- a/src/java/org/apache/solr/schema/DateField.java
+++ b/src/java/org/apache/solr/schema/DateField.java
@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.solr.core.SolrException;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
@@ -117,7 +116,7 @@ public class DateField extends FieldType {
     return indexedForm + 'Z';
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -129,7 +128,7 @@ public class DateField extends FieldType {
     return new OrdFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeDate(name, toExternal(f));
   }
 
diff --git a/src/java/org/apache/solr/schema/DoubleField.java b/src/java/org/apache/solr/schema/DoubleField.java
index 92ddc7dc754..323a914d399 100644
--- a/src/java/org/apache/solr/schema/DoubleField.java
+++ b/src/java/org/apache/solr/schema/DoubleField.java
@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FloatFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -47,7 +46,7 @@ public class DoubleField extends FieldType {
     return new FloatFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeDouble(name, f.stringValue());
   }
 
diff --git a/src/java/org/apache/solr/schema/FieldType.java b/src/java/org/apache/solr/schema/FieldType.java
index 6db42d5c453..9608103d90a 100644
--- a/src/java/org/apache/solr/schema/FieldType.java
+++ b/src/java/org/apache/solr/schema/FieldType.java
@@ -225,7 +225,7 @@ public abstract class FieldType extends FieldProperties {
    */
   public String toExternal(Fieldable f) {
     // currently used in writing XML of the search result (but perhaps
-    // a more efficient toXML(Field f, Writer w) should be used
+    // a more efficient toXML(Fieldable f, Writer w) should be used
     // in the future.
     return f.stringValue();
   }
@@ -357,7 +357,7 @@ public abstract class FieldType extends FieldProperties {
   /**
    * Renders the specified field as XML
    */
-  public abstract void write(XMLWriter xmlWriter, String name, Field f) throws IOException;
+  public abstract void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException;
 
   /**
    * calls back to TextResponseWriter to write the field value
diff --git a/src/java/org/apache/solr/schema/FloatField.java b/src/java/org/apache/solr/schema/FloatField.java
index 49ad8ffa86f..b68c2c590a3 100644
--- a/src/java/org/apache/solr/schema/FloatField.java
+++ b/src/java/org/apache/solr/schema/FloatField.java
@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FloatFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -44,7 +43,7 @@ public class FloatField extends FieldType {
     return new FloatFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeFloat(name, f.stringValue());
   }
 
diff --git a/src/java/org/apache/solr/schema/IndexSchema.java b/src/java/org/apache/solr/schema/IndexSchema.java
index 0b10cc6808f..957618e24bb 100644
--- a/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/src/java/org/apache/solr/schema/IndexSchema.java
@@ -20,6 +20,7 @@ package org.apache.solr.schema;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.search.DefaultSimilarity;
 import org.apache.lucene.search.Similarity;
 import org.apache.solr.core.SolrException;
@@ -170,8 +171,8 @@ public final class IndexSchema {
    * @return null if this schema has no unique key field
    * @see #printableUniqueKey
    */
-  public Field getUniqueKeyField(org.apache.lucene.document.Document doc) {
-    return doc.getField(uniqueKeyFieldName);  // this should return null if name is null
+  public Fieldable getUniqueKeyField(org.apache.lucene.document.Document doc) {
+    return doc.getFieldable(uniqueKeyFieldName);  // this should return null if name is null
   }
 
   /**
@@ -180,7 +181,7 @@ public final class IndexSchema {
    * @return null if this schema has no unique key field
    */
   public String printableUniqueKey(org.apache.lucene.document.Document doc) {
-     Field f = doc.getField(uniqueKeyFieldName);
+     Fieldable f = doc.getFieldable(uniqueKeyFieldName);
      return f==null ? null : uniqueKeyFieldType.toExternal(f);
   }
 
diff --git a/src/java/org/apache/solr/schema/IntField.java b/src/java/org/apache/solr/schema/IntField.java
index 593a83282b1..0a9eb1aa02c 100644
--- a/src/java/org/apache/solr/schema/IntField.java
+++ b/src/java/org/apache/solr/schema/IntField.java
@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.IntFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -44,7 +43,7 @@ public class IntField extends FieldType {
     return new IntFieldSource(field.name);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeInt(name, f.stringValue());
   }
 
diff --git a/src/java/org/apache/solr/schema/LongField.java b/src/java/org/apache/solr/schema/LongField.java
index 909f97c7085..fb31d937f9e 100644
--- a/src/java/org/apache/solr/schema/LongField.java
+++ b/src/java/org/apache/solr/schema/LongField.java
@@ -20,7 +20,6 @@ package org.apache.solr.schema;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.IntFieldSource;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -49,7 +48,7 @@ public class LongField extends FieldType {
   }
 
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeLong(name, f.stringValue());
   }
 
diff --git a/src/java/org/apache/solr/schema/SchemaField.java b/src/java/org/apache/solr/schema/SchemaField.java
index eb4a9251f2f..7b1d97e0199 100644
--- a/src/java/org/apache/solr/schema/SchemaField.java
+++ b/src/java/org/apache/solr/schema/SchemaField.java
@@ -93,7 +93,7 @@ public final class SchemaField extends FieldProperties {
             + "}";
   }
 
-  public void write(XMLWriter writer, String name, Field val) throws IOException {
+  public void write(XMLWriter writer, String name, Fieldable val) throws IOException {
     // name is passed in because it may be null if name should not be used.
     type.write(writer,name,val);
   }
diff --git a/src/java/org/apache/solr/schema/SortableDoubleField.java b/src/java/org/apache/solr/schema/SortableDoubleField.java
index 5f2a4a360d0..ba2cb978144 100644
--- a/src/java/org/apache/solr/schema/SortableDoubleField.java
+++ b/src/java/org/apache/solr/schema/SortableDoubleField.java
@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -51,7 +50,7 @@ public class SortableDoubleField extends FieldType {
     return NumberUtils.double2sortableStr(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -59,7 +58,7 @@ public class SortableDoubleField extends FieldType {
     return NumberUtils.SortableStr2doubleStr(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     xmlWriter.writeDouble(name, NumberUtils.SortableStr2double(sval));
   }
diff --git a/src/java/org/apache/solr/schema/SortableFloatField.java b/src/java/org/apache/solr/schema/SortableFloatField.java
index b8b8e08f538..f6c8736fb17 100644
--- a/src/java/org/apache/solr/schema/SortableFloatField.java
+++ b/src/java/org/apache/solr/schema/SortableFloatField.java
@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -51,7 +50,7 @@ public class SortableFloatField extends FieldType {
     return NumberUtils.float2sortableStr(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -59,7 +58,7 @@ public class SortableFloatField extends FieldType {
     return NumberUtils.SortableStr2floatStr(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     xmlWriter.writeFloat(name, NumberUtils.SortableStr2float(sval));
   }
diff --git a/src/java/org/apache/solr/schema/SortableIntField.java b/src/java/org/apache/solr/schema/SortableIntField.java
index 18359def7cc..1c1e6d54132 100644
--- a/src/java/org/apache/solr/schema/SortableIntField.java
+++ b/src/java/org/apache/solr/schema/SortableIntField.java
@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -54,7 +53,7 @@ public class SortableIntField extends FieldType {
     return NumberUtils.int2sortableStr(val);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
@@ -62,7 +61,7 @@ public class SortableIntField extends FieldType {
     return NumberUtils.SortableStr2int(indexedForm);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     // since writeInt an int instead of a String since that may be more efficient
     // in the future (saves the construction of one String)
diff --git a/src/java/org/apache/solr/schema/SortableLongField.java b/src/java/org/apache/solr/schema/SortableLongField.java
index 10d461598d3..4c543a6a3fa 100644
--- a/src/java/org/apache/solr/schema/SortableLongField.java
+++ b/src/java/org/apache/solr/schema/SortableLongField.java
@@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
 import org.apache.solr.search.function.ValueSource;
 import org.apache.solr.search.function.FieldCacheSource;
 import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
 import org.apache.solr.util.NumberUtils;
@@ -55,11 +54,11 @@ public class SortableLongField extends FieldType {
     return NumberUtils.SortableStr2long(indexedForm);
   }
 
-  public String toExternal(Field f) {
+  public String toExternal(Fieldable f) {
     return indexedToReadable(f.stringValue());
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     String sval = f.stringValue();
     xmlWriter.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length()));
   }
diff --git a/src/java/org/apache/solr/schema/StrField.java b/src/java/org/apache/solr/schema/StrField.java
index 9ca2790ab57..f84ee3d0414 100644
--- a/src/java/org/apache/solr/schema/StrField.java
+++ b/src/java/org/apache/solr/schema/StrField.java
@@ -18,7 +18,6 @@
 package org.apache.solr.schema;
 
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -38,7 +37,7 @@ public class StrField extends CompressableField {
     return getStringSort(field,reverse);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeStr(name, f.stringValue());
   }
 
diff --git a/src/java/org/apache/solr/schema/TextField.java b/src/java/org/apache/solr/schema/TextField.java
index 8c61d4ef5c5..2066e717417 100644
--- a/src/java/org/apache/solr/schema/TextField.java
+++ b/src/java/org/apache/solr/schema/TextField.java
@@ -18,7 +18,6 @@
 package org.apache.solr.schema;
 
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.request.XMLWriter;
 import org.apache.solr.request.TextResponseWriter;
@@ -41,7 +40,7 @@ public class TextField extends CompressableField {
     return new SortField(field.name,SortField.STRING, reverse);
   }
 
-  public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+  public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
     xmlWriter.writeStr(name, f.stringValue());
   }
 
diff --git a/src/java/org/apache/solr/search/SolrIndexSearcher.java b/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 060577d9a55..f402d8ad46d 100644
--- a/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -17,7 +17,7 @@
 
 package org.apache.solr.search;
 
-import org.apache.lucene.document.Document;
+import org.apache.lucene.document.*;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermDocs;
@@ -29,6 +29,7 @@ import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrInfoMBean;
 import org.apache.solr.core.SolrInfoRegistry;
 import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
 import org.apache.solr.util.NamedList;
 import org.apache.solr.util.OpenBitSet;
 
@@ -299,22 +300,90 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
     return searcher.docFreq(term);
   }
 
+  /* ********************** Document retrieval *************************/
+   
+  /* Future optimizations (yonik)
+   *
+   * If no cache is present:
+   *   - use NO_LOAD instead of LAZY_LOAD
+   *   - use LOAD_AND_BREAK if a single field is begin retrieved
+   */
+
+  /**
+   * FieldSelector which loads the specified fields, and load all other
+   * field lazily.
+   */
+  class SetNonLazyFieldSelector implements FieldSelector {
+    private Set<String> fieldsToLoad;
+    SetNonLazyFieldSelector(Set<String> toLoad) {
+      fieldsToLoad = toLoad;
+    }
+    public FieldSelectorResult accept(String fieldName) { 
+      if(fieldsToLoad.contains(fieldName))
+        return FieldSelectorResult.LOAD; 
+      else
+        return FieldSelectorResult.LAZY_LOAD;
+    }
+  }
+
+  /* solrconfig lazyfields setting */
+  public static final boolean enableLazyFieldLoading = SolrConfig.config.getBool("query/enableLazyFieldLoading", false);
+
+  /**
+   * Retrieve the {@link Document} instance corresponding to the document id.
+   */
   public Document doc(int i) throws IOException {
+    return doc(i, null);
+  }
+  /**
+   * Retrieve the {@link Document} instance corresponding to the document id.
+   *
+   * Note: The document will have all fields accessable, but if a field
+   * filter is provided, only the provided fields will be loaded (the 
+   * remainder will be available lazily).
+   */
+  public Document doc(int i, Set<String> fields) throws IOException {
+    
     Document d;
     if (documentCache != null) {
       d = (Document)documentCache.get(i);
       if (d!=null) return d;
     }
 
-    d = searcher.doc(i);
+    if(!enableLazyFieldLoading || fields == null) {
+      d = searcher.getIndexReader().document(i);
+    } else {
+      d = searcher.getIndexReader().document(i, 
+             new SetNonLazyFieldSelector(fields));
+    }
 
     if (documentCache != null) {
-      documentCache.put(i,d);
+      documentCache.put(i, d);
     }
 
     return d;
   }
 
+  /**
+   * Takes a list of docs (the doc ids actually), and reads them into an array 
+   * of Documents.
+   */
+  public void readDocs(Document[] docs, DocList ids) throws IOException {
+    readDocs(docs, ids, null);
+  }
+  /**
+   * Takes a list of docs (the doc ids actually) and a set of fields to load,
+   * and reads them into an array of Documents.
+   */
+  public void readDocs(Document[] docs, DocList ids, Set<String> fields) throws IOException {
+    DocIterator iter = ids.iterator();
+    for (int i=0; i<docs.length; i++) {
+      docs[i] = doc(iter.nextDoc(), fields);
+    }
+  }
+
+  /* ********************** end document retrieval *************************/
+
   public int maxDoc() throws IOException {
     return searcher.maxDoc();
   }
@@ -1236,17 +1305,6 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
      return docs;
   }
 
-  /**
-   * Takes a list of docs (the doc ids actually), and reads them into an array 
-   * of Documents.
-   */
-  public void readDocs(Document[] docs, DocList ids) throws IOException {
-    DocIterator iter = ids.iterator();
-    for (int i=0; i<docs.length; i++) {
-      docs[i] = doc(iter.nextDoc());
-    }
-  }
-
 
 
   /**
diff --git a/src/java/org/apache/solr/util/HighlightingUtils.java b/src/java/org/apache/solr/util/HighlightingUtils.java
index 997c7010113..c880bf7fa3c 100644
--- a/src/java/org/apache/solr/util/HighlightingUtils.java
+++ b/src/java/org/apache/solr/util/HighlightingUtils.java
@@ -20,12 +20,15 @@ import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.solr.request.*;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
 import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.schema.SchemaField;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
@@ -215,13 +218,24 @@ public class HighlightingUtils
       SolrIndexSearcher searcher = req.getSearcher();
       NamedList fragments = new NamedList();
       String[] fieldNames = getHighlightFields(query, req, defaultFields);
+      Document[] readDocs = new Document[docs.size()];
+      {
+        // pre-fetch documents using the Searcher's doc cache
+        Set<String> fset = new HashSet<String>();
+        for(String f : fieldNames) { fset.add(f); }
+        // fetch unique key if one exists.
+        SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
+        if(null != keyField)
+          fset.add(keyField.getName());  
+        searcher.readDocs(readDocs, docs, fset);
+      }
 
+      // Highlight each document
       DocIterator iterator = docs.iterator();
       for (int i = 0; i < docs.size(); i++)
       {
          int docId = iterator.nextDoc();
-         // use the Searcher's doc cache
-         Document doc = searcher.doc(docId);
+         Document doc = readDocs[i];
          NamedList docSummaries = new NamedList();
          for (String fieldName : fieldNames)
          {
diff --git a/src/java/org/apache/solr/util/SolrPluginUtils.java b/src/java/org/apache/solr/util/SolrPluginUtils.java
index 07d48ce2771..53df5692f92 100644
--- a/src/java/org/apache/solr/util/SolrPluginUtils.java
+++ b/src/java/org/apache/solr/util/SolrPluginUtils.java
@@ -35,6 +35,7 @@ import org.apache.solr.request.SolrQueryResponse;
 import org.apache.solr.request.DefaultSolrParams;
 import org.apache.solr.request.AppendedSolrParams;
 import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.*;
 
 import java.io.IOException;
@@ -205,6 +206,57 @@ public class SolrPluginUtils {
     return flags;
   }
 
+  /**
+   * Pre-fetch documents into the index searcher's document cache.
+   *
+   * This is an entirely optional step which you might want to perform for
+   * the following reasons:
+   *
+   * <ul>
+   *     <li>Locates the document-retrieval costs in one spot, which helps
+   *     detailed performance measurement</li>
+   *   
+   *     <li>Determines a priori what fields will be needed to be fetched by
+   *     various subtasks, like response writing and highlighting.  This
+   *     minimizes the chance that many needed fields will be loaded lazily.
+   *     (it is more efficient to load all the field we require normally).</li>
+   * </ul>
+   *
+   * If lazy field loading is disabled, this method does nothing.
+   */
+  public static void optimizePreFetchDocs(DocList docs,
+                                          Query query,
+                                          SolrQueryRequest req,
+                                          SolrQueryResponse res) throws IOException {
+    SolrIndexSearcher searcher = req.getSearcher();
+    if(!searcher.enableLazyFieldLoading) {
+      // nothing to do
+      return;
+    }
+
+    Set<String> fieldFilter = null;
+    Set<String> returnFields = res.getReturnFields();
+    if(returnFields != null) {
+      // copy return fields list
+      fieldFilter = new HashSet<String>(returnFields);
+      // add highlight fields
+      if(HighlightingUtils.isHighlightingEnabled(req)) {
+        for(String field: HighlightingUtils.getHighlightFields(query, req, null)) 
+          fieldFilter.add(field);        
+      }
+      // fetch unique key if one exists.
+      SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
+      if(null != keyField)
+          fieldFilter.add(keyField.getName());  
+    }
+
+    // get documents
+    DocIterator iter = docs.iterator();
+    for (int i=0; i<docs.size(); i++) {
+      searcher.doc(iter.nextDoc(), fieldFilter);
+    }
+  }
+
   /**
    * <p>
    * Returns a NamedList containing many "standard" pieces of debugging
@@ -794,6 +846,7 @@ public class SolrPluginUtils {
     }
             
   }
+
 }
 
 /** 
diff --git a/src/test/org/apache/solr/BasicFunctionalityTest.java b/src/test/org/apache/solr/BasicFunctionalityTest.java
index 8af250c0710..6a123915f86 100644
--- a/src/test/org/apache/solr/BasicFunctionalityTest.java
+++ b/src/test/org/apache/solr/BasicFunctionalityTest.java
@@ -17,9 +17,10 @@
 
 package org.apache.solr;
 
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.*;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.solr.core.SolrCore;
 import org.apache.solr.search.*;
 import org.apache.solr.request.*;
 import org.apache.solr.util.*;
@@ -568,6 +569,48 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase {
     
   }
 
+  public void testNotLazyField() throws IOException {
+    for(int i = 0; i < 10; i++) {
+      assertU(adoc("id", new Integer(i).toString(), 
+                   "title", "keyword",
+                   "test_hlt", mkstr(20000)));
+    }
+    assertU(commit());
+    SolrCore core = h.getCore();
+   
+    SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title,test_hlt");
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    core.execute(req, rsp);
+
+    DocList dl = (DocList) rsp.getValues().get(null);
+    org.apache.lucene.document.Document d = req.getSearcher().doc(dl.iterator().nextDoc());
+    // ensure field is not lazy
+    assertTrue( d.getFieldable("test_hlt") instanceof Field );
+    assertTrue( d.getFieldable("title") instanceof Field );
+  }
+
+  public void testLazyField() throws IOException {
+    for(int i = 0; i < 10; i++) {
+      assertU(adoc("id", new Integer(i).toString(), 
+                   "title", "keyword",
+                   "test_hlt", mkstr(20000)));
+    }
+    assertU(commit());
+    SolrCore core = h.getCore();
+    
+    SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title");
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    core.execute(req, rsp);
+
+    DocList dl = (DocList) rsp.getValues().get(null);
+    DocIterator di = dl.iterator();    
+    org.apache.lucene.document.Document d = req.getSearcher().doc(di.nextDoc());
+    // ensure field is lazy
+    assertTrue( !( d.getFieldable("test_hlt") instanceof Field ) );
+    assertTrue( d.getFieldable("title") instanceof Field );
+  } 
+            
+
   /** @see org.apache.solr.util.DateMathParserTest */
   public void testDateMath() {
 
diff --git a/src/test/test-files/solr/conf/solrconfig.xml b/src/test/test-files/solr/conf/solrconfig.xml
index 724976a6dcb..3964f18c36c 100644
--- a/src/test/test-files/solr/conf/solrconfig.xml
+++ b/src/test/test-files/solr/conf/solrconfig.xml
@@ -106,7 +106,12 @@
       initialSize="512"
       autowarmCount="0"/>
 
+    <!-- If true, stored fields that are not requested will be loaded lazily.
+    -->
+    <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
     <!--
+
     <cache name="myUserCache"
       class="solr.search.LRUCache"
       size="4096"