Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-12722: [child] transformer now processes 'fl'

This commit is contained in:
David Smiley 2018-09-05 10:50:14 -04:00
parent c8b47e2024
commit e4f256be15
8 changed files with 116 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
solr/CHANGES.txt
View File

@ -204,6 +204,10 @@ New Features
* SOLR-12716: NodeLostTrigger should support deleting replicas from lost nodes by setting preferredOperation=deletenode.
(shalin)
* SOLR-12722: The [child] transformer now takes an 'fl' param to specify which fields to return. It will evaluate
doc transformers if present. In 7.5 a missing 'fl' defaults to the current behavior of all fields, but in 8.0
defaults to the top/request "fl". (Moshe Bla, David Smiley)
Bug Fixes
----------------------

18
solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
View File

@ -54,17 +54,16 @@ class ChildDocTransformer extends DocTransformer {
private final DocSet childDocSet;
private final int limit;
private final boolean isNestedSchema;
private final SolrReturnFields childReturnFields;
//TODO ought to be provided/configurable
private final SolrReturnFields childReturnFields = new SolrReturnFields();
ChildDocTransformer(String name, BitSetProducer parentsFilter,
DocSet childDocSet, boolean isNestedSchema, int limit) {
ChildDocTransformer(String name, BitSetProducer parentsFilter, DocSet childDocSet,
SolrReturnFields returnFields, boolean isNestedSchema, int limit) {
this.name = name;
this.parentsFilter = parentsFilter;
this.childDocSet = childDocSet;
this.limit = limit;
this.isNestedSchema = isNestedSchema;
this.childReturnFields = returnFields!=null? returnFields: new SolrReturnFields();
}
@Override
@ -72,6 +71,9 @@ class ChildDocTransformer extends DocTransformer {
return name;
}
@Override
public boolean needsSolrIndexSearcher() { return true; }
@Override
public void transform(SolrDocument rootDoc, int rootDocId) {
// note: this algorithm works if both if we have have _nest_path_ and also if we don't!
@ -132,6 +134,12 @@ class ChildDocTransformer extends DocTransformer {
// load the doc
SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, childReturnFields);
if(childReturnFields.getTransformer() != null) {
if(childReturnFields.getTransformer().context == null) {
childReturnFields.getTransformer().setContext(context);
}
childReturnFields.getTransformer().transform(doc, docId);
}
if (isAncestor) {
// if this path has pending child docs, add them.

30
solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java
View File

@ -33,6 +33,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.search.SyntaxError;
import static org.apache.solr.schema.IndexSchema.NEST_PATH_FIELD_NAME;
@ -57,12 +58,28 @@ public class ChildDocTransformerFactory extends TransformerFactory {
static final char PATH_SEP_CHAR = '/';
static final char NUM_SEP_CHAR = '#';
private static final ThreadLocal<Boolean> recursionCheckThreadLocal = ThreadLocal.withInitial(() -> Boolean.FALSE);
private static final BooleanQuery rootFilter = new BooleanQuery.Builder()
.add(new BooleanClause(new MatchAllDocsQuery(), BooleanClause.Occur.MUST))
.add(new BooleanClause(new DocValuesFieldExistsQuery(NEST_PATH_FIELD_NAME), BooleanClause.Occur.MUST_NOT)).build();
@Override
public DocTransformer create(String field, SolrParams params, SolrQueryRequest req) {
if(recursionCheckThreadLocal.get()) {
// this is a recursive call by SolrReturnFields, see ChildDocTransformerFactory#createChildDocTransformer
return new DocTransformer.NoopFieldTransformer();
} else {
try {
// transformer is yet to be initialized in this thread, create it
recursionCheckThreadLocal.set(true);
return createChildDocTransformer(field, params, req);
} finally {
recursionCheckThreadLocal.set(false);
}
}
}
private DocTransformer createChildDocTransformer(String field, SolrParams params, SolrQueryRequest req) {
SchemaField uniqueKeyField = req.getSchema().getUniqueKeyField();
if (uniqueKeyField == null) {
throw new SolrException( ErrorCode.BAD_REQUEST,
@ -106,9 +123,20 @@ public class ChildDocTransformerFactory extends TransformerFactory {
}
}
String childReturnFields = params.get("fl");
SolrReturnFields childSolrReturnFields;
if(childReturnFields != null) {
childSolrReturnFields = new SolrReturnFields(childReturnFields, req);
} else if(req.getSchema().getDefaultLuceneMatchVersion().major < 8) {
// ensure backwards for versions prior to SOLR 8
childSolrReturnFields = new SolrReturnFields();
} else {
childSolrReturnFields = new SolrReturnFields(req);
}
int limit = params.getInt( "limit", 10 );
return new ChildDocTransformer(field, parentsFilter, childDocSet, buildHierarchy, limit);
return new ChildDocTransformer(field, parentsFilter, childDocSet, childSolrReturnFields, buildHierarchy, limit);
}
private static Query parseQuery(String qstr, SolrQueryRequest req, String param) {

23
solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
View File

@ -114,4 +114,27 @@ public abstract class DocTransformer {
public String toString() {
return getName();
}
/**
* Trivial Impl that ensure that the specified field is requested as an "extra" field,
* but then does nothing during the transformation phase.
*/
public static final class NoopFieldTransformer extends DocTransformer {
final String field;
public NoopFieldTransformer() {
this.field = null;
}
public NoopFieldTransformer(String field ) {
this.field = field;
}
public String getName() { return "noop"; }
public String[] getExtraRequestFields() {
return this.field==null? null: new String[] { field };
}
public void transform(SolrDocument doc, int docid) {
// No-Op
}
}
}

20
solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
View File

@ -84,28 +84,10 @@ public class RawValueTransformerFactory extends TransformerFactory
if (field.equals(display)) {
// we have to ensure the field is returned
return new NoopFieldTransformer(field);
return new DocTransformer.NoopFieldTransformer(field);
}
return new RenameFieldTransformer( field, display, false );
}
/**
* Trivial Impl that ensure that the specified field is requested as an "extra" field,
* but then does nothing during the transformation phase.
*/
private static final class NoopFieldTransformer extends DocTransformer {
final String field;
public NoopFieldTransformer(String field ) {
this.field = field;
}
public String getName() { return "noop"; }
public String[] getExtraRequestFields() {
return new String[] { field };
}
public void transform(SolrDocument doc, int docid) {
// No-Op
}
}
static class RawTransformer extends DocTransformer
{

51
solr/core/src/test/org/apache/solr/response/transform/TestChildDocTransformer.java
View File

@ -16,9 +16,15 @@
*/
package org.apache.solr.response.transform;
import java.util.Collection;
import java.util.Iterator;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.BasicResultContext;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@ -61,6 +67,7 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
testSubQueryJSON();
testChildDocNonStoredDVFields();
testChildReturnFields();
}
private void testChildDoctransformerXML() throws Exception {
@ -91,10 +98,10 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
"fl", "*,[child parentFilter=\"subject:parentDocument\"]"), test1);
assertQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
"fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
assertQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
"fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
}
private void testSubQueryXML() {
@ -212,10 +219,10 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
"fl", "*,[child parentFilter=\"subject:parentDocument\"]"), test1);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
"fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=3]"), test3);
"fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=3]"), test3);
}
private void testChildDocNonStoredDVFields() throws Exception {
@ -243,13 +250,41 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
"fl", "*,[child parentFilter=\"subject:parentDocument\"]"), test1);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
"fl", "intDvoDefault, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
"fl", "intDvoDefault, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
}
private void testChildReturnFields() throws Exception {
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "*,[child parentFilter=\"subject:parentDocument\" fl=\"intDvoDefault,child_fl:[value v='child_fl_test']\"]"),
"/response/docs/[0]/intDefault==42",
"/response/docs/[0]/_childDocuments_/[0]/intDvoDefault==42",
"/response/docs/[0]/_childDocuments_/[0]/child_fl=='child_fl_test'");
try(SolrQueryRequest req = req("q", "*:*", "fq", "subject:\"parentDocument\" ",
"fl", "intDefault,[child parentFilter=\"subject:parentDocument\" fl=\"intDvoDefault, [docid]\"]")) {
BasicResultContext res = (BasicResultContext) h.queryAndResponse("/select", req).getResponse();
Iterator<SolrDocument> docsStreamer = res.getProcessedDocuments();
while (docsStreamer.hasNext()) {
SolrDocument doc = docsStreamer.next();
assertFalse("root docs should not contain fields specified in child return fields", doc.containsKey("intDvoDefault"));
assertTrue("root docs should contain fields specified in query return fields", doc.containsKey("intDefault"));
Collection<SolrDocument> childDocs = doc.getChildDocuments();
for(SolrDocument childDoc: childDocs) {
assertEquals("child doc should only have 2 keys", 2, childDoc.keySet().size());
assertTrue("child docs should contain fields specified in child return fields", childDoc.containsKey("intDvoDefault"));
assertEquals("child docs should contain fields specified in child return fields",
42, childDoc.getFieldValue("intDvoDefault"));
assertTrue("child docs should contain fields specified in child return fields", childDoc.containsKey("[docid]"));
}
}
}
}
private void createSimpleIndex() {
SolrInputDocument parentDocument = new SolrInputDocument();
@ -339,7 +374,7 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
assertJQ(req("q", "*:*",
"sort", "id asc",
"fq", "subject:\"parentDocument\" ",
"fl", "id,[child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
"fl", "id, cat, title, [child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
tests);
}
@ -398,7 +433,7 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
assertQ(req("q", "*:*",
"sort", "id asc",
"fq", "subject:\"parentDocument\" ",
"fl", "id,[child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
"fl", "id, cat, title, [child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
tests);
}

1
solr/solr-ref-guide/src/transforming-result-documents.adoc
View File

@ -137,6 +137,7 @@ When using this transformer, the `parentFilter` parameter must be specified, and
* `childFilter` - query to filter which child documents should be included, this can be particularly useful when you have multiple levels of hierarchical documents (default: all children)
* `limit` - the maximum number of child documents to be returned per parent document (default: 10)
* `fl` - the field list which the transformer is to return (default: top level "fl"). There is a further limitation in which the fields here should be a subset of those specified by the top level fl param
=== [shard] - ShardAugmenterFactory

4
solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
View File

@ -1831,7 +1831,7 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
q = new SolrQuery("q", "*:*", "indent", "true");
q.setFilterQueries(parentFilter);
q.setFields("id,[child parentFilter=\"" + parentFilter +
q.setFields("id, level_i, [child parentFilter=\"" + parentFilter +
"\" childFilter=\"" + childFilter +
"\" limit=\"" + maxKidCount + "\"], name");
resp = client.query(q);
@ -1916,7 +1916,7 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
q = new SolrQuery("q", "name:" + name, "indent", "true");
}
q.setFilterQueries(parentFilter);
q.setFields("id,[child parentFilter=\"" + parentFilter +
q.setFields("id, level_i, [child parentFilter=\"" + parentFilter +
"\" childFilter=\"" + childFilter +
"\" limit=\"" + maxKidCount + "\"],name");
resp = client.query(q);