Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-52 lazyfields patch implemented. 

I am going to commit this with solrconfig defaulting to disabling lazyfields and use it in my 
application.  When I'm satisfied with the behaviour, I'll close SOLR-52 and perhaps enable 
lazyfields as the default option.



git-svn-id: https://svn.apache.org/repos/asf/incubator/solr/trunk@479793 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mike Klaas 2006-11-27 22:40:21 +00:00
parent 91c83248e6
commit c982fdedd6
30 changed files with 244 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -88,6 +88,8 @@ Changes in runtime behavior
(hossman, SOLR-25)
8. Document update handling locking is much sparser, allowing performance gains
through multiple threads. Large commits also might be faster (klaas, SOLR-65)
9. Lazy field loading can be enabled via a solrconfig directive. This will be faster when
not all stored fields are needed from a document (klaas, SOLR-52)
Optimizations
1. getDocListAndSet can now generate both a DocList and a DocSet from a

4
example/solr/conf/solrconfig.xml
View File

@ -132,6 +132,10 @@
initialSize="512"
autowarmCount="0"/>
<!-- If true, stored fields that are not requested will be loaded lazily.
-->
<enableLazyFieldLoading>false</enableLazyFieldLoading>
<!-- Example of a generic cache. These caches may be accessed by name
through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
The purpose is to enable easy caching of user/application level data.

3
src/java/org/apache/solr/request/DisMaxRequestHandler.java
View File

@ -347,6 +347,9 @@ public class DisMaxRequestHandler
flags);
}
rsp.add("search-results",results.docList);
// pre-fetch returned documents
U.optimizePreFetchDocs(results.docList, query, req, rsp);
if (null != facetInfo) rsp.add("facet_counts", facetInfo);

2
src/java/org/apache/solr/request/JSONResponseWriter.java
View File

@ -382,7 +382,7 @@ class JSONWriter extends TextResponseWriter {
DocIterator iterator = ids.iterator();
for (int i=0; i<sz; i++) {
int id = iterator.nextDoc();
Document doc = searcher.doc(id);
Document doc = searcher.doc(id, fields);
if (first) {
first=false;

5
src/java/org/apache/solr/request/StandardRequestHandler.java
View File

@ -136,6 +136,9 @@ public class StandardRequestHandler implements SolrRequestHandler, SolrInfoMBean
flags);
}
// pre-fetch returned documents
U.optimizePreFetchDocs(results.docList, query, req, rsp);
rsp.add(null,results.docList);
if (null != facetInfo) rsp.add("facet_counts", facetInfo);
@ -154,7 +157,7 @@ public class StandardRequestHandler implements SolrRequestHandler, SolrInfoMBean
rsp.add("debug", dbg);
}
} catch (Exception e) {
SolrException.logOnce(SolrCore.log, "Exception durring debug", e);
SolrException.logOnce(SolrCore.log, "Exception during debug", e);
rsp.add("exception_during_debug", SolrException.toStr(e));
}

17
src/java/org/apache/solr/request/XMLWriter.java
View File

@ -30,7 +30,7 @@ import java.io.Writer;
import java.io.IOException;
import java.util.*;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Document;
/**
* @author yonik
@ -215,7 +215,7 @@ final public class XMLWriter {
private static final Comparator fieldnameComparator = new Comparator() {
public int compare(Object o, Object o1) {
Field f1 = (Field)o; Field f2 = (Field)o1;
Fieldable f1 = (Fieldable)o; Fieldable f2 = (Fieldable)o1;
int cmp = f1.name().compareTo(f2.name());
return cmp;
// note - the sort is stable, so this should not have affected the ordering
@ -238,13 +238,12 @@ final public class XMLWriter {
// an array. The fastest way to detect multiple fields
// with the same name is to sort them first.
Enumeration ee = doc.fields();
// using global tlst here, so we shouldn't call any other
// function that uses it until we are done.
tlst.clear();
while (ee.hasMoreElements()) {
Field ff = (Field) ee.nextElement();
for (Object obj : doc.getFields()) {
Fieldable ff = (Fieldable)obj;
// skip this field if it is not a field to be returned.
if (returnFields!=null && !returnFields.contains(ff.name())) {
continue;
@ -256,12 +255,12 @@ final public class XMLWriter {
int sz = tlst.size();
int fidx1 = 0, fidx2 = 0;
while (fidx1 < sz) {
Field f1 = (Field)tlst.get(fidx1);
Fieldable f1 = (Fieldable)tlst.get(fidx1);
String fname = f1.name();
// find the end of fields with this name
fidx2 = fidx1+1;
while (fidx2 < sz && fname.equals(((Field)tlst.get(fidx2)).name()) ) {
while (fidx2 < sz && fname.equals(((Fieldable)tlst.get(fidx2)).name()) ) {
fidx2++;
}
@ -297,7 +296,7 @@ final public class XMLWriter {
indent();
cnt=0;
}
sf.write(this, null, (Field)tlst.get(i));
sf.write(this, null, (Fieldable)tlst.get(i));
}
decLevel();
// if (doIndent) indent();
@ -343,7 +342,7 @@ final public class XMLWriter {
DocIterator iterator = ids.iterator();
for (int i=0; i<sz; i++) {
int id = iterator.nextDoc();
Document doc = searcher.doc(id);
Document doc = searcher.doc(id, fields);
writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
}
decLevel();

5
src/java/org/apache/solr/schema/BCDIntField.java
View File

@ -19,7 +19,6 @@ package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.util.BCDUtils;
import org.apache.solr.request.XMLWriter;
@ -47,7 +46,7 @@ public class BCDIntField extends FieldType {
return BCDUtils.base10toBase10kSortableInt(val);
}
public String toExternal(Field f) {
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@ -55,7 +54,7 @@ public class BCDIntField extends FieldType {
return BCDUtils.base10kSortableIntToBase10(indexedForm);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeInt(name,toExternal(f));
}

4
src/java/org/apache/solr/schema/BCDLongField.java
View File

@ -18,7 +18,7 @@
package org.apache.solr.schema;
import org.apache.solr.request.XMLWriter;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import java.io.IOException;
/**
@ -26,7 +26,7 @@ import java.io.IOException;
* @version $Id$
*/
public class BCDLongField extends BCDIntField {
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeLong(name,toExternal(f));
}
}

4
src/java/org/apache/solr/schema/BCDStrField.java
View File

@ -18,7 +18,7 @@
package org.apache.solr.schema;
import org.apache.solr.request.XMLWriter;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import java.io.IOException;
/**
@ -26,7 +26,7 @@ import java.io.IOException;
* @version $Id$
*/
public class BCDStrField extends BCDIntField {
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeStr(name,toExternal(f));
}
}

5
src/java/org/apache/solr/schema/BoolField.java
View File

@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@ -86,7 +85,7 @@ public class BoolField extends FieldType {
return (ch=='1' || ch=='t' || ch=='T') ? "T" : "F";
}
public String toExternal(Field f) {
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@ -95,7 +94,7 @@ public class BoolField extends FieldType {
return ch=='T' ? "true" : "false";
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeBool(name, f.stringValue().charAt(0) =='T');
}

1
src/java/org/apache/solr/schema/CompressableField.java
View File

@ -18,6 +18,7 @@
package org.apache.solr.schema;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.*;

5
src/java/org/apache/solr/schema/DateField.java
View File

@ -20,7 +20,6 @@ package org.apache.solr.schema;
import org.apache.solr.core.SolrException;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
@ -117,7 +116,7 @@ public class DateField extends FieldType {
return indexedForm + 'Z';
}
public String toExternal(Field f) {
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@ -129,7 +128,7 @@ public class DateField extends FieldType {
return new OrdFieldSource(field.name);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeDate(name, toExternal(f));
}

3
src/java/org/apache/solr/schema/DoubleField.java
View File

@ -20,7 +20,6 @@ package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FloatFieldSource;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@ -47,7 +46,7 @@ public class DoubleField extends FieldType {
return new FloatFieldSource(field.name);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeDouble(name, f.stringValue());
}

4
src/java/org/apache/solr/schema/FieldType.java
View File

@ -225,7 +225,7 @@ public abstract class FieldType extends FieldProperties {
*/
public String toExternal(Fieldable f) {
// currently used in writing XML of the search result (but perhaps
// a more efficient toXML(Field f, Writer w) should be used
// a more efficient toXML(Fieldable f, Writer w) should be used
// in the future.
return f.stringValue();
}
@ -357,7 +357,7 @@ public abstract class FieldType extends FieldProperties {
/**
* Renders the specified field as XML
*/
public abstract void write(XMLWriter xmlWriter, String name, Field f) throws IOException;
public abstract void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException;
/**
* calls back to TextResponseWriter to write the field value

3
src/java/org/apache/solr/schema/FloatField.java
View File

@ -20,7 +20,6 @@ package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FloatFieldSource;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@ -44,7 +43,7 @@ public class FloatField extends FieldType {
return new FloatFieldSource(field.name);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeFloat(name, f.stringValue());
}

7
src/java/org/apache/solr/schema/IndexSchema.java
View File

@ -20,6 +20,7 @@ package org.apache.solr.schema;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Similarity;
import org.apache.solr.core.SolrException;
@ -170,8 +171,8 @@ public final class IndexSchema {
* @return null if this schema has no unique key field
* @see #printableUniqueKey
*/
public Field getUniqueKeyField(org.apache.lucene.document.Document doc) {
return doc.getField(uniqueKeyFieldName); // this should return null if name is null
public Fieldable getUniqueKeyField(org.apache.lucene.document.Document doc) {
return doc.getFieldable(uniqueKeyFieldName); // this should return null if name is null
}
/**
@ -180,7 +181,7 @@ public final class IndexSchema {
* @return null if this schema has no unique key field
*/
public String printableUniqueKey(org.apache.lucene.document.Document doc) {
Field f = doc.getField(uniqueKeyFieldName);
Fieldable f = doc.getFieldable(uniqueKeyFieldName);
return f==null ? null : uniqueKeyFieldType.toExternal(f);
}

3
src/java/org/apache/solr/schema/IntField.java
View File

@ -20,7 +20,6 @@ package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.IntFieldSource;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@ -44,7 +43,7 @@ public class IntField extends FieldType {
return new IntFieldSource(field.name);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeInt(name, f.stringValue());
}

3
src/java/org/apache/solr/schema/LongField.java
View File

@ -20,7 +20,6 @@ package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.IntFieldSource;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@ -49,7 +48,7 @@ public class LongField extends FieldType {
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeLong(name, f.stringValue());
}

2
src/java/org/apache/solr/schema/SchemaField.java
View File

@ -93,7 +93,7 @@ public final class SchemaField extends FieldProperties {
+ "}";
}
public void write(XMLWriter writer, String name, Field val) throws IOException {
public void write(XMLWriter writer, String name, Fieldable val) throws IOException {
// name is passed in because it may be null if name should not be used.
type.write(writer,name,val);
}

5
src/java/org/apache/solr/schema/SortableDoubleField.java
View File

@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -51,7 +50,7 @@ public class SortableDoubleField extends FieldType {
return NumberUtils.double2sortableStr(val);
}
public String toExternal(Field f) {
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@ -59,7 +58,7 @@ public class SortableDoubleField extends FieldType {
return NumberUtils.SortableStr2doubleStr(indexedForm);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
xmlWriter.writeDouble(name, NumberUtils.SortableStr2double(sval));
}

5
src/java/org/apache/solr/schema/SortableFloatField.java
View File

@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -51,7 +50,7 @@ public class SortableFloatField extends FieldType {
return NumberUtils.float2sortableStr(val);
}
public String toExternal(Field f) {
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@ -59,7 +58,7 @@ public class SortableFloatField extends FieldType {
return NumberUtils.SortableStr2floatStr(indexedForm);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
xmlWriter.writeFloat(name, NumberUtils.SortableStr2float(sval));
}

5
src/java/org/apache/solr/schema/SortableIntField.java
View File

@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -54,7 +53,7 @@ public class SortableIntField extends FieldType {
return NumberUtils.int2sortableStr(val);
}
public String toExternal(Field f) {
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@ -62,7 +61,7 @@ public class SortableIntField extends FieldType {
return NumberUtils.SortableStr2int(indexedForm);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
// since writeInt an int instead of a String since that may be more efficient
// in the future (saves the construction of one String)

5
src/java/org/apache/solr/schema/SortableLongField.java
View File

@ -22,7 +22,6 @@ import org.apache.lucene.search.FieldCache;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@ -55,11 +54,11 @@ public class SortableLongField extends FieldType {
return NumberUtils.SortableStr2long(indexedForm);
}
public String toExternal(Field f) {
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
xmlWriter.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length()));
}

3
src/java/org/apache/solr/schema/StrField.java
View File

@ -18,7 +18,6 @@
package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@ -38,7 +37,7 @@ public class StrField extends CompressableField {
return getStringSort(field,reverse);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeStr(name, f.stringValue());
}

3
src/java/org/apache/solr/schema/TextField.java
View File

@ -18,7 +18,6 @@
package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@ -41,7 +40,7 @@ public class TextField extends CompressableField {
return new SortField(field.name,SortField.STRING, reverse);
}
public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeStr(name, f.stringValue());
}

86
src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -17,7 +17,7 @@
package org.apache.solr.search;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.*;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
@ -29,6 +29,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.core.SolrInfoRegistry;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.NamedList;
import org.apache.solr.util.OpenBitSet;
@ -299,22 +300,90 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
return searcher.docFreq(term);
}
/* ********************** Document retrieval *************************/
/* Future optimizations (yonik)
*
* If no cache is present:
* - use NO_LOAD instead of LAZY_LOAD
* - use LOAD_AND_BREAK if a single field is begin retrieved
*/
/**
* FieldSelector which loads the specified fields, and load all other
* field lazily.
*/
class SetNonLazyFieldSelector implements FieldSelector {
private Set<String> fieldsToLoad;
SetNonLazyFieldSelector(Set<String> toLoad) {
fieldsToLoad = toLoad;
}
public FieldSelectorResult accept(String fieldName) {
if(fieldsToLoad.contains(fieldName))
return FieldSelectorResult.LOAD;
else
return FieldSelectorResult.LAZY_LOAD;
}
}
/* solrconfig lazyfields setting */
public static final boolean enableLazyFieldLoading = SolrConfig.config.getBool("query/enableLazyFieldLoading", false);
/**
* Retrieve the {@link Document} instance corresponding to the document id.
*/
public Document doc(int i) throws IOException {
return doc(i, null);
}
/**
* Retrieve the {@link Document} instance corresponding to the document id.
*
* Note: The document will have all fields accessable, but if a field
* filter is provided, only the provided fields will be loaded (the
* remainder will be available lazily).
*/
public Document doc(int i, Set<String> fields) throws IOException {
Document d;
if (documentCache != null) {
d = (Document)documentCache.get(i);
if (d!=null) return d;
}
d = searcher.doc(i);
if(!enableLazyFieldLoading || fields == null) {
d = searcher.getIndexReader().document(i);
} else {
d = searcher.getIndexReader().document(i,
new SetNonLazyFieldSelector(fields));
}
if (documentCache != null) {
documentCache.put(i,d);
documentCache.put(i, d);
}
return d;
}
/**
* Takes a list of docs (the doc ids actually), and reads them into an array
* of Documents.
*/
public void readDocs(Document[] docs, DocList ids) throws IOException {
readDocs(docs, ids, null);
}
/**
* Takes a list of docs (the doc ids actually) and a set of fields to load,
* and reads them into an array of Documents.
*/
public void readDocs(Document[] docs, DocList ids, Set<String> fields) throws IOException {
DocIterator iter = ids.iterator();
for (int i=0; i<docs.length; i++) {
docs[i] = doc(iter.nextDoc(), fields);
}
}
/* ********************** end document retrieval *************************/
public int maxDoc() throws IOException {
return searcher.maxDoc();
}
@ -1236,17 +1305,6 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
return docs;
}
/**
* Takes a list of docs (the doc ids actually), and reads them into an array
* of Documents.
*/
public void readDocs(Document[] docs, DocList ids) throws IOException {
DocIterator iter = ids.iterator();
for (int i=0; i<docs.length; i++) {
docs[i] = doc(iter.nextDoc());
}
}
/**

18
src/java/org/apache/solr/util/HighlightingUtils.java
View File

@ -20,12 +20,15 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.solr.request.*;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.schema.SchemaField;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
@ -215,13 +218,24 @@ public class HighlightingUtils
SolrIndexSearcher searcher = req.getSearcher();
NamedList fragments = new NamedList();
String[] fieldNames = getHighlightFields(query, req, defaultFields);
Document[] readDocs = new Document[docs.size()];
{
// pre-fetch documents using the Searcher's doc cache
Set<String> fset = new HashSet<String>();
for(String f : fieldNames) { fset.add(f); }
// fetch unique key if one exists.
SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
if(null != keyField)
fset.add(keyField.getName());
searcher.readDocs(readDocs, docs, fset);
}
// Highlight each document
DocIterator iterator = docs.iterator();
for (int i = 0; i < docs.size(); i++)
{
int docId = iterator.nextDoc();
// use the Searcher's doc cache
Document doc = searcher.doc(docId);
Document doc = readDocs[i];
NamedList docSummaries = new NamedList();
for (String fieldName : fieldNames)
{

53
src/java/org/apache/solr/util/SolrPluginUtils.java
View File

@ -35,6 +35,7 @@ import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.request.DefaultSolrParams;
import org.apache.solr.request.AppendedSolrParams;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.*;
import java.io.IOException;
@ -205,6 +206,57 @@ public class SolrPluginUtils {
return flags;
}
/**
* Pre-fetch documents into the index searcher's document cache.
*
* This is an entirely optional step which you might want to perform for
* the following reasons:
*
* <ul>
* <li>Locates the document-retrieval costs in one spot, which helps
* detailed performance measurement</li>
*
* <li>Determines a priori what fields will be needed to be fetched by
* various subtasks, like response writing and highlighting. This
* minimizes the chance that many needed fields will be loaded lazily.
* (it is more efficient to load all the field we require normally).</li>
* </ul>
*
* If lazy field loading is disabled, this method does nothing.
*/
public static void optimizePreFetchDocs(DocList docs,
Query query,
SolrQueryRequest req,
SolrQueryResponse res) throws IOException {
SolrIndexSearcher searcher = req.getSearcher();
if(!searcher.enableLazyFieldLoading) {
// nothing to do
return;
}
Set<String> fieldFilter = null;
Set<String> returnFields = res.getReturnFields();
if(returnFields != null) {
// copy return fields list
fieldFilter = new HashSet<String>(returnFields);
// add highlight fields
if(HighlightingUtils.isHighlightingEnabled(req)) {
for(String field: HighlightingUtils.getHighlightFields(query, req, null))
fieldFilter.add(field);
}
// fetch unique key if one exists.
SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
if(null != keyField)
fieldFilter.add(keyField.getName());
}
// get documents
DocIterator iter = docs.iterator();
for (int i=0; i<docs.size(); i++) {
searcher.doc(iter.nextDoc(), fieldFilter);
}
}
/**
* <p>
* Returns a NamedList containing many "standard" pieces of debugging
@ -794,6 +846,7 @@ public class SolrPluginUtils {
}
}
}
/**

45
src/test/org/apache/solr/BasicFunctionalityTest.java
View File

@ -17,9 +17,10 @@
package org.apache.solr;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.*;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.BooleanQuery;
import org.apache.solr.core.SolrCore;
import org.apache.solr.search.*;
import org.apache.solr.request.*;
import org.apache.solr.util.*;
@ -568,6 +569,48 @@ public class BasicFunctionalityTest extends AbstractSolrTestCase {
}
public void testNotLazyField() throws IOException {
for(int i = 0; i < 10; i++) {
assertU(adoc("id", new Integer(i).toString(),
"title", "keyword",
"test_hlt", mkstr(20000)));
}
assertU(commit());
SolrCore core = h.getCore();
SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title,test_hlt");
SolrQueryResponse rsp = new SolrQueryResponse();
core.execute(req, rsp);
DocList dl = (DocList) rsp.getValues().get(null);
org.apache.lucene.document.Document d = req.getSearcher().doc(dl.iterator().nextDoc());
// ensure field is not lazy
assertTrue( d.getFieldable("test_hlt") instanceof Field );
assertTrue( d.getFieldable("title") instanceof Field );
}
public void testLazyField() throws IOException {
for(int i = 0; i < 10; i++) {
assertU(adoc("id", new Integer(i).toString(),
"title", "keyword",
"test_hlt", mkstr(20000)));
}
assertU(commit());
SolrCore core = h.getCore();
SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title");
SolrQueryResponse rsp = new SolrQueryResponse();
core.execute(req, rsp);
DocList dl = (DocList) rsp.getValues().get(null);
DocIterator di = dl.iterator();
org.apache.lucene.document.Document d = req.getSearcher().doc(di.nextDoc());
// ensure field is lazy
assertTrue( !( d.getFieldable("test_hlt") instanceof Field ) );
assertTrue( d.getFieldable("title") instanceof Field );
}
/** @see org.apache.solr.util.DateMathParserTest */
public void testDateMath() {

5
src/test/test-files/solr/conf/solrconfig.xml
View File

@ -106,7 +106,12 @@
initialSize="512"
autowarmCount="0"/>
<!-- If true, stored fields that are not requested will be loaded lazily.
-->
<enableLazyFieldLoading>true</enableLazyFieldLoading>
<!--
<cache name="myUserCache"
class="solr.search.LRUCache"
size="4096"