From b3de5c989cdd276ccfdb9206d9011c789e999987 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Fri, 24 Mar 2017 23:01:32 -0400 Subject: [PATCH] SOLR-10304: Refactor new SolrDocumentFetcher out of SolrIndexSearcher --- solr/CHANGES.txt | 3 + .../clustering/ClusteringComponent.java | 64 +- .../solr/collection1/conf/schema.xml | 11 +- .../clustering/ClusteringComponentTest.java | 68 ++- .../carrot2/CarrotClusteringEngineTest.java | 3 +- .../component/RealTimeGetComponent.java | 22 +- .../solr/highlight/SolrHighlighter.java | 2 +- .../apache/solr/response/DocsStreamer.java | 104 ++-- .../solr/response/TextResponseWriter.java | 2 +- .../transform/ChildDocTransformerFactory.java | 2 +- .../solr/search/SolrDocumentFetcher.java | 571 +++++++++++++++++ .../apache/solr/search/SolrIndexSearcher.java | 573 ++---------------- .../TopGroupsResultTransformer.java | 6 +- .../org/apache/solr/util/SolrPluginUtils.java | 7 +- .../apache/solr/search/LargeFieldTest.java | 8 +- .../apache/solr/util/SolrPluginUtilsTest.java | 89 +-- 16 files changed, 867 insertions(+), 668 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index f396f808322..e2a5eb679e6 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -146,6 +146,9 @@ Other Changes * SOLR-10249: Refactor IndexFetcher.doFetch() to return a more detailed result. (Jeff Miller via David Smiley) +* SOLR-10304: Refactor Document handling out of SolrIndexSearcher into a new class "SolrDocumentFetcher". + Deprecated SolrPluginUtils.docListToSolrDocumentList(). (David Smiley) + ================== 6.5.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java index 6275c906f02..7c13e6b6630 100644 --- a/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java +++ b/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexableField; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; @@ -39,8 +41,12 @@ import org.apache.solr.handler.clustering.carrot2.CarrotClusteringEngine; import org.apache.solr.handler.component.ResponseBuilder; import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.handler.component.ShardRequest; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; import org.apache.solr.search.DocListAndSet; -import org.apache.solr.util.SolrPluginUtils; +import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +93,60 @@ public class ClusteringComponent extends SearchComponent implements SolrCoreAwar */ private NamedList initParams; + /** + * Convert a DocList to a SolrDocumentList + * + * The optional param "ids" is populated with the lucene document id + * for each SolrDocument. + * + * @param docs The {@link org.apache.solr.search.DocList} to convert + * @param searcher The {@link org.apache.solr.search.SolrIndexSearcher} to use to load the docs from the Lucene index + * @param fields The names of the Fields to load + * @param ids A map to store the ids of the docs + * @return The new {@link SolrDocumentList} containing all the loaded docs + * @throws IOException if there was a problem loading the docs + * @since solr 1.4 + */ + public static SolrDocumentList docListToSolrDocumentList( + DocList docs, + SolrIndexSearcher searcher, + Set fields, + Map ids ) throws IOException + { + IndexSchema schema = searcher.getSchema(); + + SolrDocumentList list = new SolrDocumentList(); + list.setNumFound(docs.matches()); + list.setMaxScore(docs.maxScore()); + list.setStart(docs.offset()); + + DocIterator dit = docs.iterator(); + + while (dit.hasNext()) { + int docid = dit.nextDoc(); + + Document luceneDoc = searcher.doc(docid, fields); + SolrDocument doc = new SolrDocument(); + + for( IndexableField field : luceneDoc) { + if (null == fields || fields.contains(field.name())) { + SchemaField sf = schema.getField( field.name() ); + doc.addField( field.name(), sf.getType().toObject( field ) ); + } + } + if (docs.hasScores() && (null == fields || fields.contains("score"))) { + doc.addField("score", dit.score()); + } + + list.add( doc ); + + if( ids != null ) { + ids.put( doc, new Integer(docid) ); + } + } + return list; + } + @Override @SuppressWarnings({"rawtypes", "unchecked"}) public void init(NamedList args) { @@ -172,7 +232,7 @@ public class ClusteringComponent extends SearchComponent implements SolrCoreAwar checkAvailable(name, engine); DocListAndSet results = rb.getResults(); Map docIds = new HashMap<>(results.docList.size()); - SolrDocumentList solrDocList = SolrPluginUtils.docListToSolrDocumentList( + SolrDocumentList solrDocList = docListToSolrDocumentList( results.docList, rb.req.getSearcher(), engine.getFieldsToLoad(rb.req), docIds); Object clusters = engine.cluster(rb.getQuery(), solrDocList, docIds, rb.req); rb.rsp.add("clusters", clusters); diff --git a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml index af59a20cf2f..0c06a48adc2 100644 --- a/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml +++ b/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml @@ -277,7 +277,7 @@ --> - + @@ -305,6 +305,10 @@ + + + + + + + + diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java index e8cf8309bb8..bff9f378930 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java @@ -15,6 +15,13 @@ * limitations under the License. */ package org.apache.solr.handler.clustering; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -24,8 +31,14 @@ import org.apache.solr.handler.component.QueryComponent; import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.DocList; +import org.apache.solr.search.QueryCommand; +import org.apache.solr.search.QueryResult; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.Before; import org.junit.Test; /** @@ -34,6 +47,11 @@ import org.junit.Test; **/ public class ClusteringComponentTest extends AbstractClusteringTestCase { + @Before + public void doBefore() { + clearIndex(); + } + @Test public void testComponent() throws Exception { SolrCore core = h.getCore(); @@ -79,4 +97,52 @@ public class ClusteringComponentTest extends AbstractClusteringTestCase { req.close(); } + + // tests ClusteringComponent.docListToSolrDocumentList + @Test + public void testDocListConversion() throws Exception { + assertU("", adoc("id", "3234", "url", "ignoreme", "val_i", "1", + "val_dynamic", "quick red fox")); + assertU("", adoc("id", "3235", "url", "ignoreme", "val_i", "1", + "val_dynamic", "quick green fox")); + assertU("", adoc("id", "3236", "url", "ignoreme", "val_i", "1", + "val_dynamic", "quick brown fox")); + assertU("", commit()); + + RefCounted holder = h.getCore().getSearcher(); + try { + SolrIndexSearcher srchr = holder.get(); + QueryResult qr = new QueryResult(); + QueryCommand cmd = new QueryCommand(); + cmd.setQuery(new MatchAllDocsQuery()); + cmd.setLen(10); + qr = srchr.search(qr, cmd); + + DocList docs = qr.getDocList(); + assertEquals("wrong docs size", 3, docs.size()); + Set fields = new HashSet<>(); + fields.add("val_dynamic"); + fields.add("dynamic_val"); + fields.add("range_facet_l"); // copied from id + + SolrDocumentList list = ClusteringComponent.docListToSolrDocumentList(docs, srchr, fields, null); + assertEquals("wrong list Size", docs.size(), list.size()); + for (SolrDocument document : list) { + + assertTrue("unexpected field", ! document.containsKey("val_i")); + assertTrue("unexpected id field", ! document.containsKey("id")); + + assertTrue("original field", document.containsKey("val_dynamic")); + assertTrue("dyn copy field", document.containsKey("dynamic_val")); + assertTrue("copy field", document.containsKey("range_facet_l")); + + assertNotNull("original field null", document.get("val_dynamic")); + assertNotNull("dyn copy field null", document.get("dynamic_val")); + assertNotNull("copy field null", document.get("range_facet_l")); + } + } finally { + if (null != holder) holder.decref(); + } + } + } diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java index 3d6f3d39e5f..6a5b8a08a76 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java @@ -42,7 +42,6 @@ import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; -import org.apache.solr.util.SolrPluginUtils; import org.carrot2.clustering.lingo.LingoClusteringAlgorithm; import org.carrot2.core.LanguageCode; import org.carrot2.util.attribute.AttributeUtils; @@ -465,7 +464,7 @@ public class CarrotClusteringEngineTest extends AbstractClusteringTestCase { // Perform clustering LocalSolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), solrParams); Map docIds = new HashMap<>(docList.size()); - SolrDocumentList solrDocList = SolrPluginUtils.docListToSolrDocumentList( docList, searcher, engine.getFieldsToLoad(req), docIds ); + SolrDocumentList solrDocList = ClusteringComponent.docListToSolrDocumentList( docList, searcher, engine.getFieldsToLoad(req), docIds ); @SuppressWarnings("unchecked") List> results = (List>) engine.cluster(query, solrDocList, docIds, req); diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java index 421e74fc917..882decb1627 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java @@ -69,6 +69,7 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.QParser; import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; @@ -290,7 +291,8 @@ public class RealTimeGetComponent extends SearchComponent Document luceneDocument = searcherInfo.getSearcher().doc(docid, rsp.getReturnFields().getLuceneFieldNames()); SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema()); - searcherInfo.getSearcher().decorateDocValueFields(doc, docid, searcherInfo.getSearcher().getNonStoredDVs(true)); + SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher(); + docFetcher.decorateDocValueFields(doc, docid, docFetcher.getNonStoredDVs(true)); if ( null != transformer) { if (null == resultContext) { // either first pass, or we've re-opened searcher - either way now we setContext @@ -423,7 +425,8 @@ public class RealTimeGetComponent extends SearchComponent } Document luceneDocument = searcher.doc(docid, returnFields.getLuceneFieldNames()); SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema()); - searcher.decorateDocValueFields(doc, docid, searcher.getNonStoredDVs(false)); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); + docFetcher.decorateDocValueFields(doc, docid, docFetcher.getNonStoredDVs(false)); return doc; } finally { @@ -471,10 +474,10 @@ public class RealTimeGetComponent extends SearchComponent } SolrDocument doc; - Set decorateFields = onlyTheseFields == null ? searcher.getNonStoredDVs(false): onlyTheseFields; + Set decorateFields = onlyTheseFields == null ? searcher.getDocFetcher().getNonStoredDVs(false): onlyTheseFields; Document luceneDocument = searcher.doc(docid, returnFields.getLuceneFieldNames()); doc = toSolrDoc(luceneDocument, core.getLatestSchema()); - searcher.decorateDocValueFields(doc, docid, decorateFields); + searcher.getDocFetcher().decorateDocValueFields(doc, docid, decorateFields); long docVersion = (long) doc.getFirstValue(VERSION_FIELD); Object partialVersionObj = partialDoc.getFieldValue(VERSION_FIELD); @@ -483,7 +486,7 @@ public class RealTimeGetComponent extends SearchComponent if (docVersion > partialDocVersion) { return doc; } - for (String fieldName: (Iterable) partialDoc.getFieldNames()) { + for (String fieldName: partialDoc.getFieldNames()) { doc.setField(fieldName.toString(), partialDoc.getFieldValue(fieldName)); // since partial doc will only contain single valued fields, this is fine } @@ -604,17 +607,18 @@ public class RealTimeGetComponent extends SearchComponent int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes)); if (docid < 0) return null; - + + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); if (avoidRetrievingStoredFields) { sid = new SolrInputDocument(); } else { - Document luceneDocument = searcher.doc(docid); + Document luceneDocument = docFetcher.doc(docid); sid = toSolrInputDocument(luceneDocument, core.getLatestSchema()); } if (onlyTheseNonStoredDVs != null) { - searcher.decorateDocValueFields(sid, docid, onlyTheseNonStoredDVs); + docFetcher.decorateDocValueFields(sid, docid, onlyTheseNonStoredDVs); } else { - searcher.decorateDocValueFields(sid, docid, searcher.getNonStoredDVsWithoutCopyTargets()); + docFetcher.decorateDocValueFields(sid, docid, docFetcher.getNonStoredDVsWithoutCopyTargets()); } } } finally { diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java index a8ee7346ac9..e526c57e52a 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java @@ -69,7 +69,7 @@ public abstract class SolrHighlighter if (fields[0].contains("*")) { // create a Java regular expression from the wildcard string String fieldRegex = fields[0].replaceAll("\\*", ".*"); - Collection storedHighlightFieldNames = request.getSearcher().getStoredHighlightFieldNames(); + Collection storedHighlightFieldNames = request.getSearcher().getDocFetcher().getStoredHighlightFieldNames(); List storedFieldsToHighlight = new ArrayList<>(); for (String storedFieldName: storedHighlightFieldNames) { if (storedFieldName.matches(fieldRegex)) { diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java index bdea9ec94b0..40e0afc7484 100644 --- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java +++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java @@ -49,6 +49,8 @@ import org.apache.solr.schema.TrieIntField; import org.apache.solr.schema.TrieLongField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrReturnFields; /** @@ -57,15 +59,17 @@ import org.apache.solr.search.SolrReturnFields; public class DocsStreamer implements Iterator { public static final Set KNOWN_TYPES = new HashSet<>(); - private org.apache.solr.response.ResultContext rctx; + private final org.apache.solr.response.ResultContext rctx; + private final SolrDocumentFetcher docFetcher; // a collaborator of SolrIndexSearcher private final DocList docs; - private DocTransformer transformer; - private DocIterator docIterator; + private final DocTransformer transformer; + private final DocIterator docIterator; + + private final Set fnames; // returnFields.getLuceneFieldNames(). Maybe null. Not empty. + private final boolean onlyPseudoFields; + private final Set dvFieldsToReturn; // maybe null. Not empty. - private boolean onlyPseudoFields; - private Set fnames; - private Set dvFieldsToReturn; private int idx = -1; public DocsStreamer(ResultContext rctx) { @@ -74,46 +78,61 @@ public class DocsStreamer implements Iterator { transformer = rctx.getReturnFields().getTransformer(); docIterator = this.docs.iterator(); fnames = rctx.getReturnFields().getLuceneFieldNames(); + //TODO move onlyPseudoFields calc to ReturnFields onlyPseudoFields = (fnames == null && !rctx.getReturnFields().wantsAllFields() && !rctx.getReturnFields().hasPatternMatching()) || (fnames != null && fnames.size() == 1 && SolrReturnFields.SCORE.equals(fnames.iterator().next())); // add non-stored DV fields that may have been requested - if (rctx.getReturnFields().wantsAllFields()) { - // check whether there are no additional fields - Set fieldNames = rctx.getReturnFields().getLuceneFieldNames(true); - if (fieldNames == null) { - dvFieldsToReturn = rctx.getSearcher().getNonStoredDVs(true); - } else { - dvFieldsToReturn = new HashSet<>(rctx.getSearcher().getNonStoredDVs(true)); // copy - // add all requested fields that may be useDocValuesAsStored=false - for (String fl : fieldNames) { - if (rctx.getSearcher().getNonStoredDVs(false).contains(fl)) { - dvFieldsToReturn.add(fl); - } - } - } - } else { - if (rctx.getReturnFields().hasPatternMatching()) { - for (String s : rctx.getSearcher().getNonStoredDVs(true)) { - if (rctx.getReturnFields().wantsField(s)) { - if (null == dvFieldsToReturn) { - dvFieldsToReturn = new HashSet<>(); - } - dvFieldsToReturn.add(s); - } - } - } else if (fnames != null) { - dvFieldsToReturn = new HashSet<>(fnames); // copy - // here we get all non-stored dv fields because even if a user has set - // useDocValuesAsStored=false in schema, he may have requested a field - // explicitly using the fl parameter - dvFieldsToReturn.retainAll(rctx.getSearcher().getNonStoredDVs(false)); - } - } + docFetcher = rctx.getSearcher().getDocFetcher(); + dvFieldsToReturn = calcDocValueFieldsForReturn(docFetcher, rctx.getReturnFields()); if (transformer != null) transformer.setContext(rctx); } + // TODO move to ReturnFields ? Or SolrDocumentFetcher ? + public static Set calcDocValueFieldsForReturn(SolrDocumentFetcher docFetcher, ReturnFields returnFields) { + Set result = null; + if (returnFields.wantsAllFields()) { + // check whether there are no additional fields + Set fieldNames = returnFields.getLuceneFieldNames(true); + if (fieldNames == null) { + result = docFetcher.getNonStoredDVs(true); + } else { + result = new HashSet<>(docFetcher.getNonStoredDVs(true)); // copy + // add all requested fields that may be useDocValuesAsStored=false + for (String fl : fieldNames) { + if (docFetcher.getNonStoredDVs(false).contains(fl)) { + result.add(fl); + } + } + } + } else { + if (returnFields.hasPatternMatching()) { + for (String s : docFetcher.getNonStoredDVs(true)) { + if (returnFields.wantsField(s)) { + if (null == result) { + result = new HashSet<>(); + } + result.add(s); + } + } + } else { + Set fnames = returnFields.getLuceneFieldNames(); + if (fnames == null) { + return null; + } + result = new HashSet<>(fnames); // copy + // here we get all non-stored dv fields because even if a user has set + // useDocValuesAsStored=false in schema, he may have requested a field + // explicitly using the fl parameter + result.retainAll(docFetcher.getNonStoredDVs(false)); + } + } + if (result != null && result.isEmpty()) { + return null; + } + return result; + } public int currentIndex() { return idx; @@ -133,12 +152,12 @@ public class DocsStreamer implements Iterator { sdoc = new SolrDocument(); } else { try { - Document doc = rctx.getSearcher().doc(id, fnames); - sdoc = getDoc(doc, rctx.getSearcher().getSchema()); // make sure to use the schema from the searcher and not the request (cross-core) + Document doc = docFetcher.doc(id, fnames); + sdoc = convertLuceneDocToSolrDoc(doc, rctx.getSearcher().getSchema()); // make sure to use the schema from the searcher and not the request (cross-core) // decorate the document with non-stored docValues fields if (dvFieldsToReturn != null) { - rctx.getSearcher().decorateDocValueFields(sdoc, id, dvFieldsToReturn); + docFetcher.decorateDocValueFields(sdoc, id, dvFieldsToReturn); } } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading document with docId " + id, e); @@ -157,7 +176,8 @@ public class DocsStreamer implements Iterator { } - public static SolrDocument getDoc(Document doc, final IndexSchema schema) { + // TODO move to SolrDocumentFetcher ? Refactor to also call docFetcher.decorateDocValueFields(...) ? + public static SolrDocument convertLuceneDocToSolrDoc(Document doc, final IndexSchema schema) { SolrDocument out = new SolrDocument(); for (IndexableField f : doc.getFields()) { // Make sure multivalued fields are represented as lists diff --git a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java index c4c29943f90..8bef94a0df1 100644 --- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java @@ -147,7 +147,7 @@ public abstract class TextResponseWriter implements PushWriter { } else if (val instanceof Date) { writeDate(name, (Date) val); } else if (val instanceof Document) { - SolrDocument doc = DocsStreamer.getDoc((Document) val, schema); + SolrDocument doc = DocsStreamer.convertLuceneDocToSolrDoc((Document) val, schema); writeSolrDocument(name, doc, returnFields, 0); } else if (val instanceof SolrDocument) { writeSolrDocument(name, (SolrDocument) val, returnFields, 0); diff --git a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java index e829e037c2d..45b0efc6632 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java @@ -139,7 +139,7 @@ class ChildDocTransformer extends DocTransformer { while(i.hasNext()) { Integer childDocNum = i.next(); Document childDoc = context.getSearcher().doc(childDocNum); - SolrDocument solrChildDoc = DocsStreamer.getDoc(childDoc, schema); + SolrDocument solrChildDoc = DocsStreamer.convertLuceneDocToSolrDoc(childDoc, schema); // TODO: future enhancement... // support an fl local param in the transformer, which is used to build diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java new file mode 100644 index 00000000000..267d4ebd560 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java @@ -0,0 +1,571 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search; + +import java.io.IOException; +import java.io.Reader; +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DocumentStoredFieldVisitor; +import org.apache.lucene.document.LazyDocument; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.apache.solr.common.SolrDocumentBase; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.schema.BoolField; +import org.apache.solr.schema.EnumField; +import org.apache.solr.schema.NumberType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TrieDateField; +import org.apache.solr.schema.TrieDoubleField; +import org.apache.solr.schema.TrieFloatField; +import org.apache.solr.schema.TrieIntField; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A helper class of {@link org.apache.solr.search.SolrIndexSearcher} for stored Document related matters + * including DocValue substitutions. + */ +public class SolrDocumentFetcher { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final SolrIndexSearcher searcher; + + private final boolean enableLazyFieldLoading; + + private final SolrCache documentCache; + + /** Contains the names/patterns of all docValues=true,stored=false fields in the schema. */ + private final Set allNonStoredDVs; + + /** Contains the names/patterns of all docValues=true,stored=false,useDocValuesAsStored=true fields in the schema. */ + private final Set nonStoredDVsUsedAsStored; + + /** Contains the names/patterns of all docValues=true,stored=false fields, excluding those that are copyField targets in the schema. */ + private final Set nonStoredDVsWithoutCopyTargets; + + private static int largeValueLengthCacheThreshold = Integer.getInteger("solr.largeField.cacheThreshold", 512 * 1024); // internal setting + + private final Set largeFields; + + private Collection storedHighlightFieldNames; // lazy populated; use getter + + SolrDocumentFetcher(SolrIndexSearcher searcher, SolrConfig solrConfig, boolean cachingEnabled) { + this.searcher = searcher; + this.enableLazyFieldLoading = solrConfig.enableLazyFieldLoading; + if (cachingEnabled) { + documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance(); + } else { + documentCache = null; + } + + final Set nonStoredDVsUsedAsStored = new HashSet<>(); + final Set allNonStoredDVs = new HashSet<>(); + final Set nonStoredDVsWithoutCopyTargets = new HashSet<>(); + final Set storedLargeFields = new HashSet<>(); + + for (FieldInfo fieldInfo : searcher.getFieldInfos()) { // can find materialized dynamic fields, unlike using the Solr IndexSchema. + final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldInfo.name); + if (schemaField == null) { + continue; + } + if (!schemaField.stored() && schemaField.hasDocValues()) { + if (schemaField.useDocValuesAsStored()) { + nonStoredDVsUsedAsStored.add(fieldInfo.name); + } + allNonStoredDVs.add(fieldInfo.name); + if (!searcher.getSchema().isCopyFieldTarget(schemaField)) { + nonStoredDVsWithoutCopyTargets.add(fieldInfo.name); + } + } + if (schemaField.stored() && schemaField.isLarge()) { + storedLargeFields.add(schemaField.getName()); + } + } + + this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored); + this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs); + this.nonStoredDVsWithoutCopyTargets = Collections.unmodifiableSet(nonStoredDVsWithoutCopyTargets); + this.largeFields = Collections.unmodifiableSet(storedLargeFields); + } + + public boolean isLazyFieldLoadingEnabled() { + return enableLazyFieldLoading; + } + + public SolrCache getDocumentCache() { + return documentCache; + } + + /** + * Returns a collection of the names of all stored fields which can be highlighted the index reader knows about. + */ + public Collection getStoredHighlightFieldNames() { + synchronized (this) { + if (storedHighlightFieldNames == null) { + storedHighlightFieldNames = new LinkedList<>(); + for (FieldInfo fieldInfo : searcher.getFieldInfos()) { + final String fieldName = fieldInfo.name; + try { + SchemaField field = searcher.getSchema().getField(fieldName); + if (field.stored() && ((field.getType() instanceof org.apache.solr.schema.TextField) + || (field.getType() instanceof org.apache.solr.schema.StrField))) { + storedHighlightFieldNames.add(fieldName); + } + } catch (RuntimeException e) { // getField() throws a SolrException, but it arrives as a RuntimeException + log.warn("Field [{}] found in index, but not defined in schema.", fieldName); + } + } + } + return storedHighlightFieldNames; + } + } + + /** @see SolrIndexSearcher#doc(int) */ + public Document doc(int docId) throws IOException { + return doc(docId, (Set) null); + } + + /** + * Retrieve the {@link Document} instance corresponding to the document id. + *

+ * NOTE: the document will have all fields accessible, but if a field filter is provided, only the provided + * fields will be loaded (the remainder will be available lazily). + * + * @see SolrIndexSearcher#doc(int, Set) + */ + public Document doc(int i, Set fields) throws IOException { + Document d; + if (documentCache != null) { + d = documentCache.get(i); + if (d != null) return d; + } + + final DirectoryReader reader = searcher.getIndexReader(); + if (documentCache != null && !enableLazyFieldLoading) { + // we do not filter the fields in this case because that would return an incomplete document which would + // be eventually cached. The alternative would be to read the stored fields twice; once with the fields + // and then without for caching leading to a performance hit + // see SOLR-8858 for related discussion + fields = null; + } + final SolrDocumentStoredFieldVisitor visitor = new SolrDocumentStoredFieldVisitor(fields, reader, i); + reader.document(i, visitor); + d = visitor.getDocument(); + + if (documentCache != null) { + documentCache.put(i, d); + } + + return d; + } + + /** {@link StoredFieldVisitor} which loads the specified fields eagerly (or all if null). + * If {@link #enableLazyFieldLoading} then the rest get special lazy field entries. Designated "large" + * fields will always get a special field entry. */ + private class SolrDocumentStoredFieldVisitor extends DocumentStoredFieldVisitor { + private final Document doc; + private final LazyDocument lazyFieldProducer; // arguably a better name than LazyDocument; at least how we use it here + private final int docId; + private final boolean addLargeFieldsLazily; + + SolrDocumentStoredFieldVisitor(Set toLoad, IndexReader reader, int docId) { + super(toLoad); + this.docId = docId; + this.doc = getDocument(); + this.lazyFieldProducer = toLoad != null && enableLazyFieldLoading ? new LazyDocument(reader, docId) : null; + this.addLargeFieldsLazily = (documentCache != null && !largeFields.isEmpty()); + //TODO can we return Status.STOP after a val is loaded and we know there are no other fields of interest? + // When: toLoad is one single-valued field, no lazyFieldProducer + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + Status status = super.needsField(fieldInfo); + assert status != Status.STOP : "Status.STOP not supported or expected"; + if (addLargeFieldsLazily && largeFields.contains(fieldInfo.name)) { // load "large" fields using this lazy mechanism + if (lazyFieldProducer != null || status == Status.YES) { + doc.add(new LargeLazyField(fieldInfo.name, docId)); + } + return Status.NO; + } + if (status == Status.NO && lazyFieldProducer != null) { // lazy + doc.add(lazyFieldProducer.getField(fieldInfo)); + } + return status; + } + } + + /** @see SolrIndexSearcher#doc(int, StoredFieldVisitor) */ + public void doc(int docId, StoredFieldVisitor visitor) throws IOException { + if (documentCache != null) { + Document cached = documentCache.get(docId); + if (cached != null) { + visitFromCached(cached, visitor); + return; + } + } + searcher.getIndexReader().document(docId, visitor); + } + + /** Executes a stored field visitor against a hit from the document cache */ + private void visitFromCached(Document document, StoredFieldVisitor visitor) throws IOException { + for (IndexableField f : document) { + final FieldInfo info = searcher.getFieldInfos().fieldInfo(f.name()); + final StoredFieldVisitor.Status needsField = visitor.needsField(info); + if (needsField == StoredFieldVisitor.Status.STOP) return; + if (needsField == StoredFieldVisitor.Status.NO) continue; + BytesRef binaryValue = f.binaryValue(); + if (binaryValue != null) { + visitor.binaryField(info, toByteArrayUnwrapIfPossible(binaryValue)); + continue; + } + Number numericValue = f.numericValue(); + if (numericValue != null) { + if (numericValue instanceof Double) { + visitor.doubleField(info, numericValue.doubleValue()); + } else if (numericValue instanceof Integer) { + visitor.intField(info, numericValue.intValue()); + } else if (numericValue instanceof Float) { + visitor.floatField(info, numericValue.floatValue()); + } else if (numericValue instanceof Long) { + visitor.longField(info, numericValue.longValue()); + } else { + throw new AssertionError(); + } + continue; + } + // must be String + if (f instanceof LargeLazyField) { // optimization to avoid premature string conversion + visitor.stringField(info, toByteArrayUnwrapIfPossible(((LargeLazyField) f).readBytes())); + } else { + visitor.stringField(info, f.stringValue().getBytes(StandardCharsets.UTF_8)); + } + } + } + + private byte[] toByteArrayUnwrapIfPossible(BytesRef bytesRef) { + if (bytesRef.offset == 0 && bytesRef.bytes.length == bytesRef.length) { + return bytesRef.bytes; + } else { + return Arrays.copyOfRange(bytesRef.bytes, bytesRef.offset, bytesRef.offset + bytesRef.length); + } + } + + /** Unlike LazyDocument.LazyField, we (a) don't cache large values, and (b) provide access to the byte[]. */ + class LargeLazyField implements IndexableField { + + final String name; + final int docId; + // synchronize on 'this' to access: + BytesRef cachedBytes; // we only conditionally populate this if it's big enough + + private LargeLazyField(String name, int docId) { + this.name = name; + this.docId = docId; + } + + @Override + public String toString() { + return fieldType().toString() + "<" + name() + ">"; // mimic Field.java + } + + @Override + public String name() { + return name; + } + + @Override + public IndexableFieldType fieldType() { + return searcher.getSchema().getField(name()); + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) { + return analyzer.tokenStream(name(), stringValue()); // or we could throw unsupported exception? + } + /** (for tests) */ + synchronized boolean hasBeenLoaded() { + return cachedBytes != null; + } + + @Override + public synchronized String stringValue() { + try { + return readBytes().utf8ToString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + synchronized BytesRef readBytes() throws IOException { + if (cachedBytes != null) { + return cachedBytes; + } else { + BytesRef bytesRef = new BytesRef(); + searcher.getIndexReader().document(docId, new StoredFieldVisitor() { + boolean done = false; + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (done) { + return Status.STOP; + } + return fieldInfo.name.equals(name()) ? Status.YES : Status.NO; + } + + @Override + public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + bytesRef.bytes = value; + bytesRef.length = value.length; + done = true; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + throw new UnsupportedOperationException("'large' binary fields are not (yet) supported"); + } + }); + if (bytesRef.length < largeValueLengthCacheThreshold) { + return cachedBytes = bytesRef; + } else { + return bytesRef; + } + } + } + + @Override + public BytesRef binaryValue() { + return null; + } + + @Override + public Reader readerValue() { + return null; + } + + @Override + public Number numericValue() { + return null; + } + } + + /** + * This will fetch and add the docValues fields to a given SolrDocument/SolrInputDocument + * + * @param doc + * A SolrDocument or SolrInputDocument instance where docValues will be added + * @param docid + * The lucene docid of the document to be populated + * @param fields + * The list of docValues fields to be decorated + */ + public void decorateDocValueFields(@SuppressWarnings("rawtypes") SolrDocumentBase doc, int docid, Set fields) + throws IOException { + final List leafContexts = searcher.getLeafContexts(); + final int subIndex = ReaderUtil.subIndex(docid, leafContexts); + final int localId = docid - leafContexts.get(subIndex).docBase; + final LeafReader leafReader = leafContexts.get(subIndex).reader(); + for (String fieldName : fields) { + final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldName); + if (schemaField == null || !schemaField.hasDocValues() || doc.containsKey(fieldName)) { + log.warn("Couldn't decorate docValues for field: [{}], schemaField: [{}]", fieldName, schemaField); + continue; + } + FieldInfo fi = searcher.getFieldInfos().fieldInfo(fieldName); + if (fi == null) { + continue; // Searcher doesn't have info about this field, hence ignore it. + } + final DocValuesType dvType = fi.getDocValuesType(); + switch (dvType) { + case NUMERIC: + final NumericDocValues ndv = leafReader.getNumericDocValues(fieldName); + if (ndv == null) { + continue; + } + Long val; + if (ndv.advanceExact(localId)) { + val = ndv.longValue(); + } else { + continue; + } + Object newVal = val; + if (schemaField.getType().isPointField()) { + // TODO: Maybe merge PointField with TrieFields here + NumberType type = schemaField.getType().getNumberType(); + switch (type) { + case INTEGER: + newVal = val.intValue(); + break; + case LONG: + newVal = val.longValue(); + break; + case FLOAT: + newVal = Float.intBitsToFloat(val.intValue()); + break; + case DOUBLE: + newVal = Double.longBitsToDouble(val); + break; + case DATE: + newVal = new Date(val); + break; + default: + throw new AssertionError("Unexpected PointType: " + type); + } + } else { + if (schemaField.getType() instanceof TrieIntField) { + newVal = val.intValue(); + } else if (schemaField.getType() instanceof TrieFloatField) { + newVal = Float.intBitsToFloat(val.intValue()); + } else if (schemaField.getType() instanceof TrieDoubleField) { + newVal = Double.longBitsToDouble(val); + } else if (schemaField.getType() instanceof TrieDateField) { + newVal = new Date(val); + } else if (schemaField.getType() instanceof EnumField) { + newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue()); + } + } + doc.addField(fieldName, newVal); + break; + case BINARY: + BinaryDocValues bdv = leafReader.getBinaryDocValues(fieldName); + if (bdv == null) { + continue; + } + BytesRef value; + if (bdv.advanceExact(localId)) { + value = BytesRef.deepCopyOf(bdv.binaryValue()); + } else { + continue; + } + doc.addField(fieldName, value); + break; + case SORTED: + SortedDocValues sdv = leafReader.getSortedDocValues(fieldName); + if (sdv == null) { + continue; + } + if (sdv.advanceExact(localId)) { + final BytesRef bRef = sdv.binaryValue(); + // Special handling for Boolean fields since they're stored as 'T' and 'F'. + if (schemaField.getType() instanceof BoolField) { + doc.addField(fieldName, schemaField.getType().toObject(schemaField, bRef)); + } else { + doc.addField(fieldName, bRef.utf8ToString()); + } + } + break; + case SORTED_NUMERIC: + final SortedNumericDocValues numericDv = leafReader.getSortedNumericDocValues(fieldName); + NumberType type = schemaField.getType().getNumberType(); + if (numericDv != null) { + if (numericDv.advance(localId) == localId) { + final List outValues = new ArrayList(numericDv.docValueCount()); + for (int i = 0; i < numericDv.docValueCount(); i++) { + long number = numericDv.nextValue(); + switch (type) { + case INTEGER: + outValues.add((int)number); + break; + case LONG: + outValues.add(number); + break; + case FLOAT: + outValues.add(NumericUtils.sortableIntToFloat((int)number)); + break; + case DOUBLE: + outValues.add(NumericUtils.sortableLongToDouble(number)); + break; + case DATE: + outValues.add(new Date(number)); + break; + default: + throw new AssertionError("Unexpected PointType: " + type); + } + } + assert outValues.size() > 0; + doc.addField(fieldName, outValues); + } + } + case SORTED_SET: + final SortedSetDocValues values = leafReader.getSortedSetDocValues(fieldName); + if (values != null && values.getValueCount() > 0) { + if (values.advance(localId) == localId) { + final List outValues = new LinkedList<>(); + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + value = values.lookupOrd(ord); + outValues.add(schemaField.getType().toObject(schemaField, value)); + } + assert outValues.size() > 0; + doc.addField(fieldName, outValues); + } + } + case NONE: + break; + } + } + } + + /** + * Returns an unmodifiable set of non-stored docValues field names. + * + * @param onlyUseDocValuesAsStored + * If false, returns all non-stored docValues. If true, returns only those non-stored docValues which have + * the {@link SchemaField#useDocValuesAsStored()} flag true. + */ + public Set getNonStoredDVs(boolean onlyUseDocValuesAsStored) { + return onlyUseDocValuesAsStored ? nonStoredDVsUsedAsStored : allNonStoredDVs; + } + + /** + * Returns an unmodifiable set of names of non-stored docValues fields, except those that are targets of a copy field. + */ + public Set getNonStoredDVsWithoutCopyTargets() { + return nonStoredDVsWithoutCopyTargets; + } + +} diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 83df60f2a42..4207a9b411b 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -18,19 +18,14 @@ package org.apache.solr.search; import java.io.Closeable; import java.io.IOException; -import java.io.Reader; import java.lang.invoke.MethodHandles; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -40,21 +35,26 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import com.google.common.collect.Iterables; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; -import org.apache.lucene.document.DocumentStoredFieldVisitor; -import org.apache.lucene.document.LazyDocument; -import org.apache.lucene.index.*; -import org.apache.lucene.index.StoredFieldVisitor.Status; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.ExitableDirectoryReader; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiPostingsEnum; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.NumericUtils; -import org.apache.solr.common.SolrDocumentBase; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.ModifiableSolrParams; @@ -71,15 +71,8 @@ import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.schema.BoolField; -import org.apache.solr.schema.EnumField; import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.NumberType; import org.apache.solr.schema.SchemaField; -import org.apache.solr.schema.TrieDateField; -import org.apache.solr.schema.TrieDoubleField; -import org.apache.solr.schema.TrieFloatField; -import org.apache.solr.schema.TrieIntField; import org.apache.solr.search.facet.UnInvertedField; import org.apache.solr.search.stats.StatsSource; import org.apache.solr.uninverting.UninvertingReader; @@ -107,6 +100,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI private final SolrCore core; private final IndexSchema schema; + private final SolrDocumentFetcher docFetcher; private final String name; private final Date openTime = new Date(); @@ -119,12 +113,10 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI private final int queryResultWindowSize; private final int queryResultMaxDocsCached; private final boolean useFilterForSortedQuery; - public final boolean enableLazyFieldLoading; private final boolean cachingEnabled; private final SolrCache filterCache; private final SolrCache queryResultCache; - private final SolrCache documentCache; private final SolrCache fieldValueCache; // map of generic caches - not synchronized since it's read-only after the constructor. @@ -135,21 +127,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI private final FieldInfos fieldInfos; - /** Contains the names/patterns of all docValues=true,stored=false fields in the schema. */ - private final Set allNonStoredDVs; - - /** Contains the names/patterns of all docValues=true,stored=false,useDocValuesAsStored=true fields in the schema. */ - private final Set nonStoredDVsUsedAsStored; - - /** Contains the names/patterns of all docValues=true,stored=false fields, excluding those that are copyField targets in the schema. */ - private final Set nonStoredDVsWithoutCopyTargets; - - private static int largeValueLengthCacheThreshold = Integer.getInteger("solr.largeField.cacheThreshold", 512 * 1024); // internal setting - - private final Set largeFields; - - private Collection storedHighlightFieldNames; // lazy populated; use getter - private DirectoryFactory directoryFactory; private final LeafReader leafReader; @@ -161,9 +138,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI private final NamedList readerStats; - private static DirectoryReader getReader(SolrCore core, SolrIndexConfig config, DirectoryFactory directoryFactory, - String path) throws IOException { + String path) throws IOException { final Directory dir = directoryFactory.get(path, DirContext.DEFAULT, config.lockType); try { return core.getIndexReaderFactory().newReader(dir, core); @@ -283,7 +259,9 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI this.queryResultWindowSize = solrConfig.queryResultWindowSize; this.queryResultMaxDocsCached = solrConfig.queryResultMaxDocsCached; this.useFilterForSortedQuery = solrConfig.useFilterForSortedQuery; - this.enableLazyFieldLoading = solrConfig.enableLazyFieldLoading; + + this.fieldInfos = leafReader.getFieldInfos(); + this.docFetcher = new SolrDocumentFetcher(this, solrConfig, enableCache); this.cachingEnabled = enableCache; if (cachingEnabled) { @@ -296,7 +274,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI queryResultCache = solrConfig.queryResultCacheConfig == null ? null : solrConfig.queryResultCacheConfig.newInstance(); if (queryResultCache != null) clist.add(queryResultCache); - documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance(); + SolrCache documentCache = docFetcher.getDocumentCache(); if (documentCache != null) clist.add(documentCache); if (solrConfig.userCacheConfigs.isEmpty()) { @@ -316,42 +294,11 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI } else { this.filterCache = null; this.queryResultCache = null; - this.documentCache = null; this.fieldValueCache = null; this.cacheMap = NO_GENERIC_CACHES; this.cacheList = NO_CACHES; } - final Set nonStoredDVsUsedAsStored = new HashSet<>(); - final Set allNonStoredDVs = new HashSet<>(); - final Set nonStoredDVsWithoutCopyTargets = new HashSet<>(); - final Set storedLargeFields = new HashSet<>(); - - this.fieldInfos = leafReader.getFieldInfos(); - for (FieldInfo fieldInfo : fieldInfos) { // can find materialized dynamic fields, unlike using the Solr IndexSchema. - final SchemaField schemaField = schema.getFieldOrNull(fieldInfo.name); - if (schemaField == null) { - continue; - } - if (!schemaField.stored() && schemaField.hasDocValues()) { - if (schemaField.useDocValuesAsStored()) { - nonStoredDVsUsedAsStored.add(fieldInfo.name); - } - allNonStoredDVs.add(fieldInfo.name); - if (!schema.isCopyFieldTarget(schemaField)) { - nonStoredDVsWithoutCopyTargets.add(fieldInfo.name); - } - } - if (schemaField.stored() && schemaField.isLarge()) { - storedLargeFields.add(schemaField.getName()); - } - } - - this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored); - this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs); - this.nonStoredDVsWithoutCopyTargets = Collections.unmodifiableSet(nonStoredDVsWithoutCopyTargets); - this.largeFields = Collections.unmodifiableSet(storedLargeFields); - // We already have our own filter cache setQueryCache(null); @@ -361,9 +308,21 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI assert ObjectReleaseTracker.track(this); } + public SolrDocumentFetcher getDocFetcher() { + return docFetcher; + } + + List getLeafContexts() { + return super.leafContexts; + } + + public FieldInfos getFieldInfos() { + return fieldInfos; + } + /* - * Override these two methods to provide a way to use global collection stats. - */ + * Override these two methods to provide a way to use global collection stats. + */ @Override public TermStatistics termStatistics(Term term, TermContext context) throws IOException { final SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo(); @@ -526,30 +485,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI return filterCache; } - /** - * Returns a collection of the names of all stored fields which can be highlighted the index reader knows about. - */ - public Collection getStoredHighlightFieldNames() { - synchronized (this) { - if (storedHighlightFieldNames == null) { - storedHighlightFieldNames = new LinkedList<>(); - for (FieldInfo fieldInfo : fieldInfos) { - final String fieldName = fieldInfo.name; - try { - SchemaField field = schema.getField(fieldName); - if (field.stored() && ((field.getType() instanceof org.apache.solr.schema.TextField) - || (field.getType() instanceof org.apache.solr.schema.StrField))) { - storedHighlightFieldNames.add(fieldName); - } - } catch (RuntimeException e) { // getField() throws a SolrException, but it arrives as a RuntimeException - log.warn("Field [{}] found in index, but not defined in schema.", fieldName); - } - } - } - return storedHighlightFieldNames; - } - } - // // Set default regenerators on filter and query caches if they don't have any // @@ -638,119 +573,26 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI // } // } - /* ********************** Document retrieval *************************/ - - /* - * Future optimizations (yonik) - * - * If no cache is present: - use NO_LOAD instead of LAZY_LOAD - use LOAD_AND_BREAK if a single field is being - * retrieved - */ - - /** {@link StoredFieldVisitor} which loads the specified fields eagerly (or all if null). - * If {@link #enableLazyFieldLoading} then the rest get special lazy field entries. Designated "large" - * fields will always get a special field entry. */ - private class SolrDocumentStoredFieldVisitor extends DocumentStoredFieldVisitor { - private final Document doc; - private final LazyDocument lazyFieldProducer; // arguably a better name than LazyDocument; at least how we use it here - private final int docId; - private final boolean addLargeFieldsLazily; - - SolrDocumentStoredFieldVisitor(Set toLoad, IndexReader reader, int docId) { - super(toLoad); - this.docId = docId; - this.doc = getDocument(); - this.lazyFieldProducer = toLoad != null && enableLazyFieldLoading ? new LazyDocument(reader, docId) : null; - this.addLargeFieldsLazily = (documentCache != null && !largeFields.isEmpty()); - //TODO can we return Status.STOP after a val is loaded and we know there are no other fields of interest? - // When: toLoad is one single-valued field, no lazyFieldProducer - } - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - Status status = super.needsField(fieldInfo); - assert status != Status.STOP : "Status.STOP not supported or expected"; - if (addLargeFieldsLazily && largeFields.contains(fieldInfo.name)) { // load "large" fields using this lazy mechanism - if (lazyFieldProducer != null || status == Status.YES) { - doc.add(new LargeLazyField(fieldInfo.name, docId)); - } - return Status.NO; - } - if (status == Status.NO && lazyFieldProducer != null) { // lazy - doc.add(lazyFieldProducer.getField(fieldInfo)); - } - return status; - } - } - /** * Retrieve the {@link Document} instance corresponding to the document id. + * + * @see SolrDocumentFetcher */ @Override - public Document doc(int i) throws IOException { - return doc(i, (Set) null); + public Document doc(int docId) throws IOException { + return doc(docId, (Set) null); } /** - * Visit a document's fields using a {@link StoredFieldVisitor} This method does not currently add to the Solr - * document cache. + * Visit a document's fields using a {@link StoredFieldVisitor}. + * This method does not currently add to the Solr document cache. * * @see IndexReader#document(int, StoredFieldVisitor) + * @see SolrDocumentFetcher */ @Override - public void doc(int docId, StoredFieldVisitor visitor) throws IOException { - if (documentCache != null) { - Document cached = documentCache.get(docId); - if (cached != null) { - visitFromCached(cached, visitor); - return; - } - } - getIndexReader().document(docId, visitor); - } - - /** Executes a stored field visitor against a hit from the document cache */ - private void visitFromCached(Document document, StoredFieldVisitor visitor) throws IOException { - for (IndexableField f : document) { - final FieldInfo info = fieldInfos.fieldInfo(f.name()); - final Status needsField = visitor.needsField(info); - if (needsField == Status.STOP) return; - if (needsField == Status.NO) continue; - BytesRef binaryValue = f.binaryValue(); - if (binaryValue != null) { - visitor.binaryField(info, toByteArrayUnwrapIfPossible(binaryValue)); - continue; - } - Number numericValue = f.numericValue(); - if (numericValue != null) { - if (numericValue instanceof Double) { - visitor.doubleField(info, numericValue.doubleValue()); - } else if (numericValue instanceof Integer) { - visitor.intField(info, numericValue.intValue()); - } else if (numericValue instanceof Float) { - visitor.floatField(info, numericValue.floatValue()); - } else if (numericValue instanceof Long) { - visitor.longField(info, numericValue.longValue()); - } else { - throw new AssertionError(); - } - continue; - } - // must be String - if (f instanceof LargeLazyField) { // optimization to avoid premature string conversion - visitor.stringField(info, toByteArrayUnwrapIfPossible(((LargeLazyField) f).readBytes())); - } else { - visitor.stringField(info, f.stringValue().getBytes(StandardCharsets.UTF_8)); - } - } - } - - private byte[] toByteArrayUnwrapIfPossible(BytesRef bytesRef) { - if (bytesRef.offset == 0 && bytesRef.bytes.length == bytesRef.length) { - return bytesRef.bytes; - } else { - return Arrays.copyOfRange(bytesRef.bytes, bytesRef.offset, bytesRef.offset + bytesRef.length); - } + public final void doc(int docId, StoredFieldVisitor visitor) throws IOException { + getDocFetcher().doc(docId, visitor); } /** @@ -758,328 +600,14 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI *

* NOTE: the document will have all fields accessible, but if a field filter is provided, only the provided * fields will be loaded (the remainder will be available lazily). + * + * @see SolrDocumentFetcher */ @Override - public Document doc(int i, Set fields) throws IOException { - - Document d; - if (documentCache != null) { - d = documentCache.get(i); - if (d != null) return d; - } - - final DirectoryReader reader = getIndexReader(); - if (documentCache != null && !enableLazyFieldLoading) { - // we do not filter the fields in this case because that would return an incomplete document which would - // be eventually cached. The alternative would be to read the stored fields twice; once with the fields - // and then without for caching leading to a performance hit - // see SOLR-8858 for related discussion - fields = null; - } - final SolrDocumentStoredFieldVisitor visitor = new SolrDocumentStoredFieldVisitor(fields, reader, i); - reader.document(i, visitor); - d = visitor.getDocument(); - - if (documentCache != null) { - documentCache.put(i, d); - } - - return d; + public final Document doc(int i, Set fields) throws IOException { + return getDocFetcher().doc(i, fields); } - /** Unlike LazyDocument.LazyField, we (a) don't cache large values, and (b) provide access to the byte[]. */ - class LargeLazyField implements IndexableField { - - final String name; - final int docId; - // synchronize on 'this' to access: - BytesRef cachedBytes; // we only conditionally populate this if it's big enough - - private LargeLazyField(String name, int docId) { - this.name = name; - this.docId = docId; - } - - @Override - public String toString() { - return fieldType().toString() + "<" + name() + ">"; // mimic Field.java - } - - @Override - public String name() { - return name; - } - - @Override - public IndexableFieldType fieldType() { - return schema.getField(name()); - } - - @Override - public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) { - return analyzer.tokenStream(name(), stringValue()); // or we could throw unsupported exception? - } - /** (for tests) */ - synchronized boolean hasBeenLoaded() { - return cachedBytes != null; - } - - @Override - public synchronized String stringValue() { - try { - return readBytes().utf8ToString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - synchronized BytesRef readBytes() throws IOException { - if (cachedBytes != null) { - return cachedBytes; - } else { - BytesRef bytesRef = new BytesRef(); - getIndexReader().document(docId, new StoredFieldVisitor() { - boolean done = false; - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (done) { - return Status.STOP; - } - return fieldInfo.name.equals(name()) ? Status.YES : Status.NO; - } - - @Override - public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { - bytesRef.bytes = value; - bytesRef.length = value.length; - done = true; - } - - @Override - public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { - throw new UnsupportedOperationException("'large' binary fields are not (yet) supported"); - } - }); - if (bytesRef.length < largeValueLengthCacheThreshold) { - return cachedBytes = bytesRef; - } else { - return bytesRef; - } - } - } - - @Override - public BytesRef binaryValue() { - return null; - } - - @Override - public Reader readerValue() { - return null; - } - - @Override - public Number numericValue() { - return null; - } - } - - /** - * This will fetch and add the docValues fields to a given SolrDocument/SolrInputDocument - * - * @param doc - * A SolrDocument or SolrInputDocument instance where docValues will be added - * @param docid - * The lucene docid of the document to be populated - * @param fields - * The list of docValues fields to be decorated - */ - public void decorateDocValueFields(@SuppressWarnings("rawtypes") SolrDocumentBase doc, int docid, Set fields) - throws IOException { - final int subIndex = ReaderUtil.subIndex(docid, leafContexts); - final int localId = docid - leafContexts.get(subIndex).docBase; - final LeafReader leafReader = leafContexts.get(subIndex).reader(); - for (String fieldName : fields) { - final SchemaField schemaField = schema.getFieldOrNull(fieldName); - if (schemaField == null || !schemaField.hasDocValues() || doc.containsKey(fieldName)) { - log.warn("Couldn't decorate docValues for field: [{}], schemaField: [{}]", fieldName, schemaField); - continue; - } - FieldInfo fi = fieldInfos.fieldInfo(fieldName); - if (fi == null) { - continue; // Searcher doesn't have info about this field, hence ignore it. - } - final DocValuesType dvType = fi.getDocValuesType(); - switch (dvType) { - case NUMERIC: - final NumericDocValues ndv = leafReader.getNumericDocValues(fieldName); - if (ndv == null) { - continue; - } - Long val; - if (ndv.advanceExact(localId)) { - val = ndv.longValue(); - } else { - continue; - } - Object newVal = val; - if (schemaField.getType().isPointField()) { - // TODO: Maybe merge PointField with TrieFields here - NumberType type = schemaField.getType().getNumberType(); - switch (type) { - case INTEGER: - newVal = val.intValue(); - break; - case LONG: - newVal = val; - break; - case FLOAT: - newVal = Float.intBitsToFloat(val.intValue()); - break; - case DOUBLE: - newVal = Double.longBitsToDouble(val); - break; - case DATE: - newVal = new Date(val); - break; - default: - throw new AssertionError("Unexpected PointType: " + type); - } - } else { - if (schemaField.getType() instanceof TrieIntField) { - newVal = val.intValue(); - } else if (schemaField.getType() instanceof TrieFloatField) { - newVal = Float.intBitsToFloat(val.intValue()); - } else if (schemaField.getType() instanceof TrieDoubleField) { - newVal = Double.longBitsToDouble(val); - } else if (schemaField.getType() instanceof TrieDateField) { - newVal = new Date(val); - } else if (schemaField.getType() instanceof EnumField) { - newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue()); - } - } - doc.addField(fieldName, newVal); - break; - case BINARY: - BinaryDocValues bdv = leafReader.getBinaryDocValues(fieldName); - if (bdv == null) { - continue; - } - BytesRef value; - if (bdv.advanceExact(localId)) { - value = BytesRef.deepCopyOf(bdv.binaryValue()); - } else { - continue; - } - doc.addField(fieldName, value); - break; - case SORTED: - SortedDocValues sdv = leafReader.getSortedDocValues(fieldName); - if (sdv == null) { - continue; - } - if (sdv.advanceExact(localId)) { - final BytesRef bRef = sdv.binaryValue(); - // Special handling for Boolean fields since they're stored as 'T' and 'F'. - if (schemaField.getType() instanceof BoolField) { - doc.addField(fieldName, schemaField.getType().toObject(schemaField, bRef)); - } else { - doc.addField(fieldName, bRef.utf8ToString()); - } - } - break; - case SORTED_NUMERIC: - final SortedNumericDocValues numericDv = leafReader.getSortedNumericDocValues(fieldName); - NumberType type = schemaField.getType().getNumberType(); - if (numericDv != null) { - if (numericDv.advance(localId) == localId) { - final List outValues = new ArrayList(numericDv.docValueCount()); - for (int i = 0; i < numericDv.docValueCount(); i++) { - long number = numericDv.nextValue(); - switch (type) { - case INTEGER: - outValues.add((int)number); - break; - case LONG: - outValues.add(number); - break; - case FLOAT: - outValues.add(NumericUtils.sortableIntToFloat((int)number)); - break; - case DOUBLE: - outValues.add(NumericUtils.sortableLongToDouble(number)); - break; - case DATE: - outValues.add(new Date(number)); - break; - default: - throw new AssertionError("Unexpected PointType: " + type); - } - } - assert outValues.size() > 0; - doc.addField(fieldName, outValues); - } - } - case SORTED_SET: - final SortedSetDocValues values = leafReader.getSortedSetDocValues(fieldName); - if (values != null && values.getValueCount() > 0) { - if (values.advance(localId) == localId) { - final List outValues = new LinkedList(); - for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { - value = values.lookupOrd(ord); - outValues.add(schemaField.getType().toObject(schemaField, value)); - } - assert outValues.size() > 0; - doc.addField(fieldName, outValues); - } - } - case NONE: - break; - } - } - } - - /** - * Takes a list of docs (the doc ids actually), and reads them into an array of Documents. - */ - public void readDocs(Document[] docs, DocList ids) throws IOException { - readDocs(docs, ids, null); - } - - /** - * Takes a list of docs (the doc ids actually) and a set of fields to load, and reads them into an array of Documents. - */ - public void readDocs(Document[] docs, DocList ids, Set fields) throws IOException { - final DocIterator iter = ids.iterator(); - for (int i = 0; i < docs.length; i++) { - docs[i] = doc(iter.nextDoc(), fields); - } - } - - /** - * Returns an unmodifiable set of non-stored docValues field names. - * - * @param onlyUseDocValuesAsStored - * If false, returns all non-stored docValues. If true, returns only those non-stored docValues which have - * the {@link SchemaField#useDocValuesAsStored()} flag true. - */ - public Set getNonStoredDVs(boolean onlyUseDocValuesAsStored) { - return onlyUseDocValuesAsStored ? nonStoredDVsUsedAsStored : allNonStoredDVs; - } - - /** - * Returns an unmodifiable set of names of non-stored docValues fields, except those that are targets of a copy field. - */ - public Set getNonStoredDVsWithoutCopyTargets() { - return nonStoredDVsWithoutCopyTargets; - } - - /* ********************** end document retrieval *************************/ - - //////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////// - /** expert: internal API, subject to change */ public SolrCache getFieldValueCache() { return fieldValueCache; @@ -2555,15 +2083,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI return a.intersects(getDocSet(deState)); } - /** - * Takes a list of document IDs, and returns an array of Documents containing all of the stored fields. - */ - public Document[] readDocs(DocList ids) throws IOException { - final Document[] docs = new Document[ids.size()]; - readDocs(docs, ids); - return docs; - } - /** * Warm this searcher based on an old one (primarily for auto-cache warming). */ diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java index adb81de2c1d..83c81e5e9fd 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java @@ -19,12 +19,12 @@ package org.apache.solr.search.grouping.distributed.shardresultserializer; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.DocumentStoredFieldVisitor; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; @@ -285,9 +285,7 @@ public class TopGroupsResultTransformer implements ShardResultTransformer fields, Map ids ) throws IOException { + /* DWS deprecation note: + It's only called by ClusteringComponent, and I think the "ids" param aspect is a bit messy and not worth supporting. + If someone wants a similar method they can speak up and we can add a method to SolrDocumentFetcher. + */ IndexSchema schema = searcher.getSchema(); SolrDocumentList list = new SolrDocumentList(); diff --git a/solr/core/src/test/org/apache/solr/search/LargeFieldTest.java b/solr/core/src/test/org/apache/solr/search/LargeFieldTest.java index 57dc2de27af..44463191838 100644 --- a/solr/core/src/test/org/apache/solr/search/LargeFieldTest.java +++ b/solr/core/src/test/org/apache/solr/search/LargeFieldTest.java @@ -112,8 +112,8 @@ public class LargeFieldTest extends SolrTestCaseJ4 { private void assertLazyNotLoaded(Document d, String fieldName) { IndexableField field = d.getField(fieldName); if (fieldName == BIG_FIELD) { - assertTrue(field instanceof SolrIndexSearcher.LargeLazyField); - assertFalse(((SolrIndexSearcher.LargeLazyField)field).hasBeenLoaded()); + assertTrue(field instanceof SolrDocumentFetcher.LargeLazyField); + assertFalse(((SolrDocumentFetcher.LargeLazyField)field).hasBeenLoaded()); } else { assertTrue(field instanceof LazyDocument.LazyField); assertFalse(((LazyDocument.LazyField)field).hasBeenLoaded()); @@ -123,8 +123,8 @@ public class LargeFieldTest extends SolrTestCaseJ4 { private void assertLazyLoaded(Document d, String fieldName) { IndexableField field = d.getField(fieldName); if (fieldName == BIG_FIELD) { - assertTrue(field instanceof SolrIndexSearcher.LargeLazyField); - assertTrue(((SolrIndexSearcher.LargeLazyField)field).hasBeenLoaded()); + assertTrue(field instanceof SolrDocumentFetcher.LargeLazyField); + assertTrue(((SolrDocumentFetcher.LargeLazyField)field).hasBeenLoaded()); } else { assertTrue(field instanceof LazyDocument.LazyField); assertTrue(((LazyDocument.LazyField)field).hasBeenLoaded()); diff --git a/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java b/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java index fc506807630..e41484fbd32 100644 --- a/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java @@ -16,37 +16,28 @@ */ package org.apache.solr.util; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.QParser; -import org.apache.solr.search.QueryCommand; -import org.apache.solr.search.QueryResult; -import org.apache.solr.util.SolrPluginUtils.DisjunctionMaxQueryParser; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.search.DocList; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.DisjunctionMaxQuery; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.BooleanClause.Occur; -import org.junit.BeforeClass; -import org.junit.Test; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.HashMap; import java.util.Iterator; -import java.util.Set; -import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.QParser; +import org.apache.solr.util.SolrPluginUtils.DisjunctionMaxQueryParser; +import org.junit.BeforeClass; +import org.junit.Test; /** * Tests that the functions in SolrPluginUtils work as advertised. @@ -58,52 +49,6 @@ public class SolrPluginUtilsTest extends SolrTestCaseJ4 { initCore("solrconfig.xml","schema.xml"); } - @Test - public void testDocListConversion() throws Exception { - assertU("", adoc("id", "3234", "val_i", "1", - "val_dynamic", "quick red fox")); - assertU("", adoc("id", "3235", "val_i", "1", - "val_dynamic", "quick green fox")); - assertU("", adoc("id", "3236", "val_i", "1", - "val_dynamic", "quick brown fox")); - assertU("", commit()); - - RefCounted holder = h.getCore().getSearcher(); - try { - SolrIndexSearcher srchr = holder.get(); - QueryResult qr = new QueryResult(); - QueryCommand cmd = new QueryCommand(); - cmd.setQuery(new MatchAllDocsQuery()); - cmd.setLen(10); - qr = srchr.search(qr, cmd); - - DocList docs = qr.getDocList(); - assertEquals("wrong docs size", 3, docs.size()); - Set fields = new HashSet<>(); - fields.add("val_dynamic"); - fields.add("dynamic_val"); - fields.add("range_facet_l"); // copied from id - - SolrDocumentList list = SolrPluginUtils.docListToSolrDocumentList(docs, srchr, fields, null); - assertEquals("wrong list Size", docs.size(), list.size()); - for (SolrDocument document : list) { - - assertTrue("unexpected field", ! document.containsKey("val_i")); - assertTrue("unexpected id field", ! document.containsKey("id")); - - assertTrue("original field", document.containsKey("val_dynamic")); - assertTrue("dyn copy field", document.containsKey("dynamic_val")); - assertTrue("copy field", document.containsKey("range_facet_l")); - - assertNotNull("original field null", document.get("val_dynamic")); - assertNotNull("dyn copy field null", document.get("dynamic_val")); - assertNotNull("copy field null", document.get("range_facet_l")); - } - } finally { - if (null != holder) holder.decref(); - } - } - @Test public void testPartialEscape() {