diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c5ebcd60428..a9805e6ba46 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -65,6 +65,8 @@ Other Changes * SOLR-7624: Remove deprecated zkCredientialsProvider element in solrcloud section of solr.xml. (Xu Zhang, Per Steffensen, Ramkumar Aiyengar, Mark Miller) + +* SOLR-6234: Scoring for query time join (Mikhail Khludnev) ================== 5.3.0 ================== diff --git a/solr/common-build.xml b/solr/common-build.xml index a5038e761fd..337caa9c148 100644 --- a/solr/common-build.xml +++ b/solr/common-build.xml @@ -320,6 +320,7 @@ + diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index e3819e3444b..8f3ffcb22ba 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -60,6 +60,7 @@ import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.schema.TrieField; +import org.apache.solr.search.join.ScoreJoinQParserPlugin; import org.apache.solr.util.RefCounted; public class JoinQParserPlugin extends QParserPlugin { @@ -72,8 +73,17 @@ public class JoinQParserPlugin extends QParserPlugin { @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { + @Override public Query parse() throws SyntaxError { + if(localParams!=null && localParams.get(ScoreJoinQParserPlugin.SCORE)!=null){ + return new ScoreJoinQParserPlugin().createParser(qstr, localParams, params, req).parse(); + }else{ + return parseJoin(); + } + } + + Query parseJoin() throws SyntaxError { String fromField = getParam("from"); String fromIndex = getParam("fromIndex"); String toField = getParam("to"); diff --git a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java new file mode 100644 index 00000000000..9d15c075335 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.JoinUtil; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.uninverting.UninvertingReader; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.search.JoinQParserPlugin; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.util.RefCounted; + +/** + * Create a query-time join query with scoring. + * It just calls {@link JoinUtil#createJoinQuery(String, boolean, String, Query, org.apache.lucene.search.IndexSearcher, ScoreMode)}. + * It runs subordinate query and collects values of "from" field and scores, then it lookups these collected values in "to" field, and + * yields aggregated scores. + * Local parameters are similar to {@link JoinQParserPlugin} {!join} + * This plugin doesn't have own name, and is called by specifying local parameter {!join score=...}.... + * Note: this parser is invoked even if you specify score=none. + *
Example:q={!join from=manu_id_s to=id score=total}foo + * + */ +public class ScoreJoinQParserPlugin extends QParserPlugin { + + public static final String SCORE = "score"; + + static class OtherCoreJoinQuery extends SameCoreJoinQuery { + private final String fromIndex; + private final long fromCoreOpenTime; + + public OtherCoreJoinQuery(Query fromQuery, String fromField, + String fromIndex, long fromCoreOpenTime, ScoreMode scoreMode, + String toField) { + super(fromQuery, fromField, toField, scoreMode); + this.fromIndex = fromIndex; + this.fromCoreOpenTime = fromCoreOpenTime; + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); + + CoreContainer container = info.getReq().getCore().getCoreDescriptor().getCoreContainer(); + + final SolrCore fromCore = container.getCore(fromIndex); + + if (fromCore == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex); + } + RefCounted fromHolder = null; + fromHolder = fromCore.getRegisteredSearcher(); + final Query joinQuery; + try { + joinQuery = JoinUtil.createJoinQuery(fromField, true, + toField, fromQuery, fromHolder.get(), scoreMode); + } finally { + fromCore.close(); + fromHolder.decref(); + } + joinQuery.setBoost(getBoost()); + return joinQuery.rewrite(reader); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + + (int) (fromCoreOpenTime ^ (fromCoreOpenTime >>> 32)); + result = prime * result + + ((fromIndex == null) ? 0 : fromIndex.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!super.equals(obj)) return false; + if (getClass() != obj.getClass()) return false; + OtherCoreJoinQuery other = (OtherCoreJoinQuery) obj; + if (fromCoreOpenTime != other.fromCoreOpenTime) return false; + if (fromIndex == null) { + if (other.fromIndex != null) return false; + } else if (!fromIndex.equals(other.fromIndex)) return false; + return true; + } + + @Override + public String toString(String field) { + return "OtherCoreJoinQuery [fromIndex=" + fromIndex + + ", fromCoreOpenTime=" + fromCoreOpenTime + " extends " + + super.toString(field) + "]"; + } + } + + static class SameCoreJoinQuery extends Query { + protected final Query fromQuery; + protected final ScoreMode scoreMode; + protected final String fromField; + protected final String toField; + + SameCoreJoinQuery(Query fromQuery, String fromField, String toField, + ScoreMode scoreMode) { + this.fromQuery = fromQuery; + this.scoreMode = scoreMode; + this.fromField = fromField; + this.toField = toField; + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); + final Query jq = JoinUtil.createJoinQuery(fromField, true, + toField, fromQuery, info.getReq().getSearcher(), scoreMode); + jq.setBoost(getBoost()); + return jq.rewrite(reader); + } + + + @Override + public String toString(String field) { + return "SameCoreJoinQuery [fromQuery=" + fromQuery + ", fromField=" + + fromField + ", toField=" + toField + ", scoreMode=" + scoreMode + + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + + ((fromField == null) ? 0 : fromField.hashCode()); + result = prime * result + + ((fromQuery == null) ? 0 : fromQuery.hashCode()); + result = prime * result + + ((scoreMode == null) ? 0 : scoreMode.hashCode()); + result = prime * result + ((toField == null) ? 0 : toField.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!super.equals(obj)) return false; + if (getClass() != obj.getClass()) return false; + SameCoreJoinQuery other = (SameCoreJoinQuery) obj; + if (fromField == null) { + if (other.fromField != null) return false; + } else if (!fromField.equals(other.fromField)) return false; + if (fromQuery == null) { + if (other.fromQuery != null) return false; + } else if (!fromQuery.equals(other.fromQuery)) return false; + if (scoreMode != other.scoreMode) return false; + if (toField == null) { + if (other.toField != null) return false; + } else if (!toField.equals(other.toField)) return false; + return true; + } + } + + final static Map lowercase = Collections.unmodifiableMap( new HashMap() { + { + for (ScoreMode s : ScoreMode.values()) { + put(s.name().toLowerCase(Locale.ROOT), s); + put(s.name(), s); + } + } + }); + + @Override + public void init(NamedList args) { + } + + + @Override + public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new QParser(qstr, localParams, params, req) { + @Override + public Query parse() throws SyntaxError { + final String fromField = localParams.get("from"); + final String fromIndex = localParams.get("fromIndex"); + final String toField = localParams.get("to"); + final ScoreMode scoreMode = parseScore(); + + final String v = localParams.get(CommonParams.VALUE); + + final Query q = createQuery(fromField, v, fromIndex, toField, scoreMode, + CommonParams.TRUE.equals(localParams.get("TESTenforceSameCoreAsAnotherOne"))); + + return q; + } + + private Query createQuery(final String fromField, final String fromQueryStr, + String fromIndex, final String toField, final ScoreMode scoreMode, + boolean byPassShortCircutCheck) throws SyntaxError { + + final String myCore = req.getCore().getCoreDescriptor().getName(); + + if (fromIndex != null && (!fromIndex.equals(myCore) || byPassShortCircutCheck)) { + CoreContainer container = req.getCore().getCoreDescriptor().getCoreContainer(); + + final SolrCore fromCore = container.getCore(fromIndex); + RefCounted fromHolder = null; + + if (fromCore == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex); + } + + long fromCoreOpenTime = 0; + LocalSolrQueryRequest otherReq = new LocalSolrQueryRequest(fromCore, params); + + try { + QParser fromQueryParser = QParser.getParser(fromQueryStr, "lucene", otherReq); + Query fromQuery = fromQueryParser.getQuery(); + + fromHolder = fromCore.getRegisteredSearcher(); + if (fromHolder != null) { + fromCoreOpenTime = fromHolder.get().getOpenTime(); + } + return new OtherCoreJoinQuery(fromQuery, fromField, fromIndex, fromCoreOpenTime, + scoreMode, toField); + } finally { + otherReq.close(); + fromCore.close(); + if (fromHolder != null) fromHolder.decref(); + } + } else { + QParser fromQueryParser = subQuery(fromQueryStr, null); + final Query fromQuery = fromQueryParser.getQuery(); + return new SameCoreJoinQuery(fromQuery, fromField, toField, scoreMode); + } + } + + private ScoreMode parseScore() { + + String score = getParam(SCORE); + final ScoreMode scoreMode = lowercase.get(score); + if (scoreMode == null) { + throw new IllegalArgumentException("Unable to parse ScoreMode from: " + score); + } + return scoreMode; + } + }; + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml new file mode 100644 index 00000000000..126551bb402 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + id + + + + + + + + diff --git a/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java new file mode 100644 index 00000000000..6ef50aed153 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr; + +import java.io.StringWriter; +import java.util.Collections; + +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.join.TestScoreJoinQPNoScore; +import org.apache.solr.servlet.DirectSolrConnection; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestCrossCoreJoin extends SolrTestCaseJ4 { + + private static SolrCore fromCore; + + @BeforeClass + public static void beforeTests() throws Exception { + System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ +// initCore("solrconfig.xml","schema12.xml"); + + // File testHome = createTempDir().toFile(); + // FileUtils.copyDirectory(getFile("solrj/solr"), testHome); + initCore("solrconfig.xml", "schema12.xml", TEST_HOME(), "collection1"); + final CoreContainer coreContainer = h.getCoreContainer(); + final CoreDescriptor toCoreDescriptor = coreContainer.getCoreDescriptor("collection1"); + final CoreDescriptor fromCoreDescriptor = new CoreDescriptor("fromCore", toCoreDescriptor) { + @Override + public String getSchemaName() { + return "schema.xml"; + } + }; + + fromCore = coreContainer.create(fromCoreDescriptor); + + assertU(add(doc("id", "1", "name", "john", "title", "Director", "dept_s", "Engineering"))); + assertU(add(doc("id", "2", "name", "mark", "title", "VP", "dept_s", "Marketing"))); + assertU(add(doc("id", "3", "name", "nancy", "title", "MTS", "dept_s", "Sales"))); + assertU(add(doc("id", "4", "name", "dave", "title", "MTS", "dept_s", "Support", "dept_s", "Engineering"))); + assertU(add(doc("id", "5", "name", "tina", "title", "VP", "dept_s", "Engineering"))); + assertU(commit()); + + update(fromCore, add(doc("id", "10", "dept_id_s", "Engineering", "text", "These guys develop stuff", "cat", "dev"))); + update(fromCore, add(doc("id", "11", "dept_id_s", "Marketing", "text", "These guys make you look good"))); + update(fromCore, add(doc("id", "12", "dept_id_s", "Sales", "text", "These guys sell stuff"))); + update(fromCore, add(doc("id", "13", "dept_id_s", "Support", "text", "These guys help customers"))); + update(fromCore, commit()); + + } + + + public static String update(SolrCore core, String xml) throws Exception { + DirectSolrConnection connection = new DirectSolrConnection(core); + SolrRequestHandler handler = core.getRequestHandler("/update"); + return connection.request(handler, null, xml); + } + + @Test + public void testJoin() throws Exception { + doTestJoin("{!join"); + } + + @Test + public void testScoreJoin() throws Exception { + doTestJoin("{!join " + TestScoreJoinQPNoScore.whateverScore()); + } + + void doTestJoin(String joinPrefix) throws Exception { + assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id", + "debugQuery", random().nextBoolean() ? "true":"false") + , "/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}" + ); + + // find people that develop stuff - but limit via filter query to a name of "john" + // this tests filters being pushed down to queries (SOLR-3062) + assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id", "fq", "name:john", + "debugQuery", random().nextBoolean() ? "true":"false") + , "/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}" + ); + } + + @Test + public void testCoresAreDifferent() throws Exception { + assertQEx("schema12.xml" + " has no \"cat\" field", req("cat:*"), ErrorCode.BAD_REQUEST); + final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, "cat:*", "lucene", 0, 100, Collections.emptyMap()); + final String resp = query(fromCore, req); + assertTrue(resp, resp.contains("numFound=\"1\"")); + assertTrue(resp, resp.contains("10")); + + } + + public String query(SolrCore core, SolrQueryRequest req) throws Exception { + String handler = "standard"; + SolrQueryResponse rsp = new SolrQueryResponse(); + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); + core.execute(core.getRequestHandler(handler), req, rsp); + if (rsp.getException() != null) { + throw rsp.getException(); + } + StringWriter sw = new StringWriter(32000); + QueryResponseWriter responseWriter = core.getQueryResponseWriter(req); + responseWriter.write(sw, req, rsp); + req.close(); + SolrRequestInfo.clearRequestInfo(); + return sw.toString(); + } + + @AfterClass + public static void nukeAll() { + fromCore = null; + } +} diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java index 837ccdcd2b1..537b8a6e01b 100644 --- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java +++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java @@ -384,6 +384,23 @@ public class QueryEqualityTest extends SolrTestCaseJ4 { } } + public void testQueryScoreJoin() throws Exception { + SolrQueryRequest req = req("myVar", "5", + "df", "text", + "ff", "foo_s", + "tt", "bar_s", + "scoreavg","avg"); + + try { + assertQueryEquals("join", req, + "{!join from=foo_s to=bar_s score=avg}asdf", + "{!join from=$ff to=$tt score=Avg}asdf", + "{!join from=$ff to='bar_s' score=$scoreavg}text:asdf"); + } finally { + req.close(); + } + } + public void testTerms() throws Exception { assertQueryEquals("terms", "{!terms f=foo_i}10,20,30,-10,-20,-30", "{!terms f=foo_i}10,20,30,-10,-20,-30"); } diff --git a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java new file mode 100644 index 00000000000..dd73deb6ac1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.solr.JSONTestUtil; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.JoinQParserPlugin; +import org.apache.solr.search.QParser; +import org.apache.solr.search.SyntaxError; +import org.junit.BeforeClass; +import org.junit.Test; +import org.noggit.JSONUtil; +import org.noggit.ObjectBuilder; + +public class TestScoreJoinQPNoScore extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeTests() throws Exception { + System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ + initCore("solrconfig-basic.xml","schema-docValuesJoin.xml"); + } + + @Test + public void testJoin() throws Exception { + assertU(add(doc("id", "1","name_s", "john", "title_s", "Director", "dept_ss","Engineering"))); + assertU(add(doc("id", "2","name_s", "mark", "title_s", "VP", "dept_ss","Marketing"))); + assertU(add(doc("id", "3","name_s", "nancy", "title_s", "MTS", "dept_ss","Sales"))); + assertU(add(doc("id", "4","name_s", "dave", "title_s", "MTS", "dept_ss","Support", "dept_ss","Engineering"))); + assertU(add(doc("id", "5","name_s", "tina", "title_s", "VP", "dept_ss","Engineering"))); + + assertU(add(doc("id","10", "dept_id_s", "Engineering", "text_t","These guys develop stuff"))); + assertU(add(doc("id","11", "dept_id_s", "Marketing", "text_t","These guys make you look good"))); + assertU(add(doc("id","12", "dept_id_s", "Sales", "text_t","These guys sell stuff"))); + assertU(add(doc("id","13", "dept_id_s", "Support", "text_t","These guys help customers"))); + + assertU(commit()); + + // test debugging TODO no debug in JoinUtil + // assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id", "debugQuery","true") + // ,"/debug/join/{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS=={'_MATCH_':'fromSetSize,toSetSize', 'fromSetSize':2, 'toSetSize':3}" + // ); + + assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id") + ,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}" + ); + + // empty from + assertJQ(req("q","{!join from=noexist_s to=dept_id_s"+whateverScore()+"}*:*", "fl","id") + ,"/response=={'numFound':0,'start':0,'docs':[]}" + ); + + // empty to + assertJQ(req("q","{!join from=dept_ss to=noexist_s"+whateverScore()+"}*:*", "fl","id") + ,"/response=={'numFound':0,'start':0,'docs':[]}" + ); + + // self join... return everyone with she same title as Dave + assertJQ(req("q","{!join from=title_s to=title_s"+whateverScore()+"}name_s:dave", "fl","id") + ,"/response=={'numFound':2,'start':0,'docs':[{'id':'3'},{'id':'4'}]}" + ); + + // find people that develop stuff + assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id") + ,"/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}" + ); + + // self join on multivalued text_t field + assertJQ(req("q","{!join from=title_s to=title_s"+whateverScore()+"}name_s:dave", "fl","id") + ,"/response=={'numFound':2,'start':0,'docs':[{'id':'3'},{'id':'4'}]}" + ); + + assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id", "debugQuery","true") + ,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}" + ); + + // expected outcome for a sub query matching dave joined against departments + final String davesDepartments = + "/response=={'numFound':2,'start':0,'docs':[{'id':'10'},{'id':'13'}]}"; + + // straight forward query + assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}name_s:dave", + "fl","id"), + davesDepartments); + + // variable deref for sub-query parsing + assertJQ(req("q","{!join from=dept_ss to=dept_id_s v=$qq"+whateverScore()+"}", + "qq","{!dismax}dave", + "qf","name_s", + "fl","id", + "debugQuery","true"), + davesDepartments); + + // variable deref for sub-query parsing w/localparams + assertJQ(req("q","{!join from=dept_ss to=dept_id_s v=$qq"+whateverScore()+"}", + "qq","{!dismax qf=name_s}dave", + "fl","id", + "debugQuery","true"), + davesDepartments); + + // defType local param to control sub-query parsing + assertJQ(req("q","{!join from=dept_ss to=dept_id_s defType=dismax"+whateverScore()+"}dave", + "qf","name_s", + "fl","id", + "debugQuery","true"), + davesDepartments); + + // find people that develop stuff - but limit via filter query to a name of "john" + // this tests filters being pushed down to queries (SOLR-3062) + assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id", "fq", "name_s:john") + ,"/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}" + ); + + + assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id" + ) + ,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}"); + + // find people that develop stuff, even if it's requested as single value + assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id") + ,"/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}" + ); + + } + + public void testJoinQueryType() throws SyntaxError, IOException{ + SolrQueryRequest req = null; + try{ + final String score = whateverScore(); + + req = req("{!join from=dept_id_s to=dept_ss"+score+"}text_t:develop"); + SolrQueryResponse rsp = new SolrQueryResponse(); + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); + + { + final Query query = QParser.getParser(req.getParams().get("q"), null, req).getQuery(); + final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader()); + assertTrue( + rewrittenQuery+" should be Lucene's", + rewrittenQuery.getClass().getPackage().getName() + .startsWith("org.apache.lucene")); + } + { + final Query query = QParser.getParser( + "{!join from=dept_id_s to=dept_ss}text_t:develop" + , null, req).getQuery(); + final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader()); + assertEquals(rewrittenQuery+" is expected to be from Solr", + JoinQParserPlugin.class.getPackage().getName(), + rewrittenQuery.getClass().getPackage().getName()); + } + }finally{ + if(req!=null){ + req.close(); + } + SolrRequestInfo.clearRequestInfo(); + } + } + + public static String whateverScore() { + final ScoreMode[] vals = ScoreMode.values(); + return " score="+vals[random().nextInt(vals.length)]+" "; + } + + @Test + public void testRandomJoin() throws Exception { + int indexIter=50 * RANDOM_MULTIPLIER; + int queryIter=50 * RANDOM_MULTIPLIER; + + // groups of fields that have any chance of matching... used to + // increase test effectiveness by avoiding 0 resultsets much of the time. + String[][] compat = new String[][] { + {"small_s_dv","small2_s_dv","small2_ss_dv","small3_ss_dv"}, + {"small_i_dv","small2_i_dv","small2_is_dv","small3_is_dv"} + }; + + + while (--indexIter >= 0) { + int indexSize = random().nextInt(20 * RANDOM_MULTIPLIER); + + List types = new ArrayList(); + types.add(new FldType("id",ONE_ONE, new SVal('A','Z',4,4))); + /** no numeric fields so far LUCENE-5868 + types.add(new FldType("score_f_dv",ONE_ONE, new FVal(1,100))); // field used to score + **/ + types.add(new FldType("small_s_dv",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1))); + types.add(new FldType("small2_s_dv",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1))); + types.add(new FldType("small2_ss_dv",ZERO_TWO, new SVal('a',(char)('c'+indexSize/3),1,1))); + types.add(new FldType("small3_ss_dv",new IRange(0,25), new SVal('A','z',1,1))); + /** no numeric fields so far LUCENE-5868 + types.add(new FldType("small_i_dv",ZERO_ONE, new IRange(0,5+indexSize/3))); + types.add(new FldType("small2_i_dv",ZERO_ONE, new IRange(0,5+indexSize/3))); + types.add(new FldType("small2_is_dv",ZERO_TWO, new IRange(0,5+indexSize/3))); + types.add(new FldType("small3_is_dv",new IRange(0,25), new IRange(0,100))); + **/ + + clearIndex(); + Map model = indexDocs(types, null, indexSize); + Map>> pivots = new HashMap>>(); + + for (int qiter=0; qiter> pivot = pivots.get(fromField+"/"+toField); + if (pivot == null) { + pivot = createJoinMap(model, fromField, toField); + pivots.put(fromField+"/"+toField, pivot); + } + + Collection fromDocs = model.values(); + Set docs = join(fromDocs, pivot); + List docList = new ArrayList(docs.size()); + for (Comparable id : docs) docList.add(model.get(id)); + Collections.sort(docList, createComparator("_docid_",true,false,false,false)); + List sortedDocs = new ArrayList(); + for (Doc doc : docList) { + if (sortedDocs.size() >= 10) break; + sortedDocs.add(doc.toObject(h.getCore().getLatestSchema())); + } + + Map resultSet = new LinkedHashMap(); + resultSet.put("numFound", docList.size()); + resultSet.put("start", 0); + resultSet.put("docs", sortedDocs); + + // todo: use different join queries for better coverage + + SolrQueryRequest req = req("wt","json","indent","true", "echoParams","all", + "q","{!join from="+fromField+" to="+toField + +" "+ (random().nextBoolean() ? "fromIndex=collection1" : "") + +" "+ (random().nextBoolean() ? "TESTenforceSameCoreAsAnotherOne=true" : "") + +" "+whateverScore()+"}*:*" + , "sort", "_docid_ asc" + ); + + String strResponse = h.query(req); + + Object realResponse = ObjectBuilder.fromJSON(strResponse); + String err = JSONTestUtil.matchObj("/response", realResponse, resultSet); + if (err != null) { + final String m = "JOIN MISMATCH: " + err + + "\n\trequest="+req + + "\n\tresult="+strResponse + + "\n\texpected="+ JSONUtil.toJSON(resultSet) + ;// + "\n\tmodel="+ JSONUtil.toJSON(model); + log.error(m); + { + SolrQueryRequest f = req("wt","json","indent","true", "echoParams","all", + "q","*:*", "facet","true", + "facet.field", fromField + , "sort", "_docid_ asc" + ,"rows","0" + ); + log.error("faceting on from field: "+h.query(f)); + } + { + final Map ps = ((MapSolrParams)req.getParams()).getMap(); + final String q = ps.get("q"); + ps.put("q", q.replaceAll("join score=none", "join")); + log.error("plain join: "+h.query(req)); + ps.put("q", q); + + } + { + // re-execute the request... good for putting a breakpoint here for debugging + final Map ps = ((MapSolrParams)req.getParams()).getMap(); + final String q = ps.get("q"); + ps.put("q", q.replaceAll("\\}", " cache=false\\}")); + String rsp = h.query(req); + } + fail(err); + } + + } + } + } + + Map> createJoinMap(Map model, String fromField, String toField) { + Map> id_to_id = new HashMap>(); + + Map> value_to_id = invertField(model, toField); + + for (Comparable fromId : model.keySet()) { + Doc doc = model.get(fromId); + List vals = doc.getValues(fromField); + if (vals == null) continue; + for (Comparable val : vals) { + List toIds = value_to_id.get(val); + if (toIds == null) continue; + Set ids = id_to_id.get(fromId); + if (ids == null) { + ids = new HashSet(); + id_to_id.put(fromId, ids); + } + for (Comparable toId : toIds) + ids.add(toId); + } + } + + return id_to_id; + } + + + Set join(Collection input, Map> joinMap) { + Set ids = new HashSet(); + for (Doc doc : input) { + Collection output = joinMap.get(doc.id); + if (output == null) continue; + ids.addAll(output); + } + return ids; + } + +} diff --git a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java new file mode 100644 index 00000000000..85fc86d6f2e --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Random; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.QParser; +import org.apache.solr.search.SolrCache; +import org.junit.BeforeClass; +import org.junit.Ignore; + +public class TestScoreJoinQPScore extends SolrTestCaseJ4 { + + private static final String idField = "id"; + private static final String toField = "movieId_s"; + + @BeforeClass + public static void beforeTests() throws Exception { + System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ + initCore("solrconfig.xml", "schema12.xml"); + } + + public void testSimple() throws Exception { + final String idField = "id"; + final String toField = "productId_s"; + + clearIndex(); + + // 0 + assertU(add(doc("t_description", "random text", + "name", "name1", + idField, "1"))); + +// 1 + + assertU(add(doc("price_s", "10.0", + idField, "2", + toField, "1"))); +// 2 + assertU(add(doc("price_s", "20.0", + idField, "3", + toField, "1"))); +// 3 + assertU(add(doc("t_description", "more random text", + "name", "name2", + idField, "4"))); +// 4 + assertU(add(doc("price_s", "10.0", + idField, "5", + toField, "4"))); +// 5 + assertU(add(doc("price_s", "20.0", + idField, "6", + toField, "4"))); + + assertU(commit()); + + // Search for product + assertJQ(req("q", "{!join from=" + idField + " to=" + toField + " score=None}name:name2", "fl", "id") + , "/response=={'numFound':2,'start':0,'docs':[{'id':'5'},{'id':'6'}]}"); + + /*Query joinQuery = + JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name2")), indexSearcher, ScoreMode.None); + + TopDocs result = indexSearcher.search(joinQuery, 10); + assertEquals(2, result.totalHits); + assertEquals(4, result.scoreDocs[0].doc); + assertEquals(5, result.scoreDocs[1].doc); + */ + assertJQ(req("q", "{!join from=" + idField + " to=" + toField + " score=None}name:name1", "fl", "id") + , "/response=={'numFound':2,'start':0,'docs':[{'id':'2'},{'id':'3'}]}"); + + /*joinQuery = JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name1")), indexSearcher, ScoreMode.None); + result = indexSearcher.search(joinQuery, 10); + assertEquals(2, result.totalHits); + assertEquals(1, result.scoreDocs[0].doc); + assertEquals(2, result.scoreDocs[1].doc);*/ + + // Search for offer + assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=None}id:5", "fl", "id") + , "/response=={'numFound':1,'start':0,'docs':[{'id':'4'}]}"); + /*joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("id", "5")), indexSearcher, ScoreMode.None); + result = indexSearcher.search(joinQuery, 10); + assertEquals(1, result.totalHits); + assertEquals(3, result.scoreDocs[0].doc); + + indexSearcher.getIndexReader().close(); + dir.close();*/ + } + + public void testSimpleWithScoring() throws Exception { + indexDataForScorring(); + + // Search for movie via subtitle + assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Max}title:random", "fl", "id") + , "/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'4'}]}"); + //dump(req("q","{!scorejoin from="+toField+" to="+idField+" score=Max}title:random", "fl","id,score", "debug", "true")); + /* + Query joinQuery = + JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "random")), indexSearcher, ScoreMode.Max); + TopDocs result = indexSearcher.search(joinQuery, 10); + assertEquals(2, result.totalHits); + assertEquals(0, result.scoreDocs[0].doc); + assertEquals(3, result.scoreDocs[1].doc);*/ + + + // Score mode max. + //dump(req("q","{!scorejoin from="+toField+" to="+idField+" score=Max}title:movie", "fl","id,score", "debug", "true")); + + // dump(req("q","title:movie", "fl","id,score", "debug", "true")); + assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Max}title:movie", "fl", "id") + , "/response=={'numFound':2,'start':0,'docs':[{'id':'4'},{'id':'1'}]}"); + + /*joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Max); + result = indexSearcher.search(joinQuery, 10); + assertEquals(2, result.totalHits); + assertEquals(3, result.scoreDocs[0].doc); + assertEquals(0, result.scoreDocs[1].doc);*/ + + // Score mode total + assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Total}title:movie", "fl", "id") + , "/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'4'}]}"); + /* joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Total); + result = indexSearcher.search(joinQuery, 10); + assertEquals(2, result.totalHits); + assertEquals(0, result.scoreDocs[0].doc); + assertEquals(3, result.scoreDocs[1].doc); +*/ + //Score mode avg + assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Avg}title:movie", "fl", "id") + , "/response=={'numFound':2,'start':0,'docs':[{'id':'4'},{'id':'1'}]}"); + + /* joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Avg); + result = indexSearcher.search(joinQuery, 10); + assertEquals(2, result.totalHits); + assertEquals(3, result.scoreDocs[0].doc); + assertEquals(0, result.scoreDocs[1].doc);*/ + + } + + final static Comparator lessFloat = new Comparator() { + @Override + public int compare(String o1, String o2) { + assertTrue(Float.parseFloat(o1) < Float.parseFloat(o2)); + return 0; + } + }; + + @Ignore("SOLR-7814, also don't forget cover boost at testCacheHit()") + public void testBoost() throws Exception { + indexDataForScorring(); + ScoreMode score = ScoreMode.values()[random().nextInt(ScoreMode.values().length)]; + + final SolrQueryRequest req = req("q", "{!join from=movieId_s to=id score=" + score + " b=200}title:movie", "fl", "id,score", "omitHeader", "true"); + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, new SolrQueryResponse())); + final Query luceneQ = QParser.getParser(req.getParams().get("q"), null, req).getQuery().rewrite(req.getSearcher().getLeafReader()); + assertEquals("" + luceneQ, Float.floatToIntBits(200), Float.floatToIntBits(luceneQ.getBoost())); + SolrRequestInfo.clearRequestInfo(); + req.close(); + } + + public void testCacheHit() throws Exception { + indexDataForScorring(); + + SolrCache cache = (SolrCache) h.getCore().getInfoRegistry() + .get("queryResultCache"); + { + final NamedList statPre = cache.getStatistics(); + h.query(req("q", "{!join from=movieId_s to=id score=Avg}title:first", "fl", "id", "omitHeader", "true")); + assertHitOrInsert(cache, statPre); + } + + { + final NamedList statPre = cache.getStatistics(); + h.query(req("q", "{!join from=movieId_s to=id score=Avg}title:first", "fl", "id", "omitHeader", "true")); + assertHit(cache, statPre); + } + + { + NamedList statPre = cache.getStatistics(); + + Random r = random(); + boolean changed = false; + boolean x = false; + String from = (x = r.nextBoolean()) ? "id" : "movieId_s"; + changed |= x; + String to = (x = r.nextBoolean()) ? "movieId_s" : "id"; + changed |= x; + String score = (x = r.nextBoolean()) ? not(ScoreMode.Avg).name() : "Avg"; + changed |= x; + /* till SOLR-7814 + * String boost = (x = r.nextBoolean()) ? "23" : "1"; + changed |= x; */ + String q = (!changed) ? (r.nextBoolean() ? "title:first^67" : "title:night") : "title:first"; + + final String resp = h.query(req("q", "{!join from=" + from + " to=" + to + + " score=" + score + + //" b=" + boost + + "}" + q, "fl", "id", "omitHeader", "true") + ); + assertInsert(cache, statPre); + + statPre = cache.getStatistics(); + final String repeat = h.query(req("q", "{!join from=" + from + " to=" + to + " score=" + score.toLowerCase(Locale.ROOT) + + //" b=" + boost + "}" + q, "fl", "id", "omitHeader", "true") + ); + assertHit(cache, statPre); + + assertEquals("lowercase shouldn't change anything", resp, repeat); + + try { + h.query(req("q", "{!join from=" + from + " to=" + to + " score=" + score.substring(0, score.length() - 1) + + "}" + q, "fl", "id", "omitHeader", "true") + ); + fail("excpecting exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("ScoreMode")); + } + } + // this queries are not overlap, with other in this test case. + // however it might be better to extract this method into the separate suite + // for a while let's nuke a cache content, in case of repetitions + cache.clear(); + } + + private ScoreMode not(ScoreMode s) { + Random r = random(); + final List l = new ArrayList(Arrays.asList(ScoreMode.values())); + l.remove(s); + return l.get(r.nextInt(l.size())); + } + + private void assertInsert(SolrCache cache, final NamedList statPre) { + assertEquals("it lookups", 1, + delta("lookups", cache.getStatistics(), statPre)); + assertEquals("it doesn't hit", 0, delta("hits", cache.getStatistics(), statPre)); + assertEquals("it inserts", 1, + delta("inserts", cache.getStatistics(), statPre)); + } + + private void assertHit(SolrCache cache, final NamedList statPre) { + assertEquals("it lookups", 1, + delta("lookups", cache.getStatistics(), statPre)); + assertEquals("it hits", 1, delta("hits", cache.getStatistics(), statPre)); + assertEquals("it doesn't insert", 0, + delta("inserts", cache.getStatistics(), statPre)); + } + + private void assertHitOrInsert(SolrCache cache, final NamedList statPre) { + assertEquals("it lookups", 1, + delta("lookups", cache.getStatistics(), statPre)); + final long mayHit = delta("hits", cache.getStatistics(), statPre); + assertTrue("it may hit", 0 == mayHit || 1 == mayHit); + assertEquals("or insert on cold", 1, + delta("inserts", cache.getStatistics(), statPre) + mayHit); + } + + private long delta(String key, NamedList a, NamedList b) { + return (Long) a.get(key) - (Long) b.get(key); + } + + private void indexDataForScorring() { + clearIndex(); +// 0 + assertU(add(doc("t_description", "A random movie", + "name", "Movie 1", + idField, "1"))); +// 1 + + assertU(add(doc("title", "The first subtitle of this movie", + idField, "2", + toField, "1"))); + + +// 2 + + assertU(add(doc("title", "random subtitle; random event movie", + idField, "3", + toField, "1"))); + +// 3 + + assertU(add(doc("t_description", "A second random movie", + "name", "Movie 2", + idField, "4"))); +// 4 + + assertU(add(doc("title", "a very random event happened during christmas night", + idField, "5", + toField, "4"))); + + +// 5 + + assertU(add(doc("title", "movie end movie test 123 test 123 random", + idField, "6", + toField, "4"))); + + + assertU(commit()); + } +}