mirror of https://github.com/apache/lucene.git
SOLR-6234: Scoring for query time join
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1693092 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mikhail Khludnev
parent
5b0e8d1d41
commit
65b595add4
|
|
@ -65,6 +65,8 @@ Other Changes
|
|||
|
||||
* SOLR-7624: Remove deprecated zkCredientialsProvider element in solrcloud section of solr.xml.
|
||||
(Xu Zhang, Per Steffensen, Ramkumar Aiyengar, Mark Miller)
|
||||
|
||||
* SOLR-6234: Scoring for query time join (Mikhail Khludnev)
|
||||
|
||||
================== 5.3.0 ==================
|
||||
|
||||
|
|
|
|||
|
|
@ -320,6 +320,7 @@
|
|||
<link offline="true" href="${lucene.javadoc.url}expressions" packagelistloc="${lucenedocs}/expressions"/>
|
||||
<link offline="true" href="${lucene.javadoc.url}suggest" packagelistloc="${lucenedocs}/suggest"/>
|
||||
<link offline="true" href="${lucene.javadoc.url}grouping" packagelistloc="${lucenedocs}/grouping"/>
|
||||
<link offline="true" href="${lucene.javadoc.url}join" packagelistloc="${lucenedocs}/join"/>
|
||||
<link offline="true" href="${lucene.javadoc.url}queries" packagelistloc="${lucenedocs}/queries"/>
|
||||
<link offline="true" href="${lucene.javadoc.url}queryparser" packagelistloc="${lucenedocs}/queryparser"/>
|
||||
<link offline="true" href="${lucene.javadoc.url}highlighter" packagelistloc="${lucenedocs}/highlighter"/>
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ import org.apache.solr.request.LocalSolrQueryRequest;
|
|||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.schema.TrieField;
|
||||
import org.apache.solr.search.join.ScoreJoinQParserPlugin;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
|
||||
public class JoinQParserPlugin extends QParserPlugin {
|
||||
|
|
@ -72,8 +73,17 @@ public class JoinQParserPlugin extends QParserPlugin {
|
|||
@Override
|
||||
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
return new QParser(qstr, localParams, params, req) {
|
||||
|
||||
@Override
|
||||
public Query parse() throws SyntaxError {
|
||||
if(localParams!=null && localParams.get(ScoreJoinQParserPlugin.SCORE)!=null){
|
||||
return new ScoreJoinQParserPlugin().createParser(qstr, localParams, params, req).parse();
|
||||
}else{
|
||||
return parseJoin();
|
||||
}
|
||||
}
|
||||
|
||||
Query parseJoin() throws SyntaxError {
|
||||
String fromField = getParam("from");
|
||||
String fromIndex = getParam("fromIndex");
|
||||
String toField = getParam("to");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,294 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.lucene.index.DocValuesType;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.join.JoinUtil;
|
||||
import org.apache.lucene.search.join.ScoreMode;
|
||||
import org.apache.lucene.uninverting.UninvertingReader;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.search.JoinQParserPlugin;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.QParserPlugin;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
|
||||
/**
|
||||
* Create a query-time join query with scoring.
|
||||
* It just calls {@link JoinUtil#createJoinQuery(String, boolean, String, Query, org.apache.lucene.search.IndexSearcher, ScoreMode)}.
|
||||
* It runs subordinate query and collects values of "from" field and scores, then it lookups these collected values in "to" field, and
|
||||
* yields aggregated scores.
|
||||
* Local parameters are similar to {@link JoinQParserPlugin} <a href="http://wiki.apache.org/solr/Join">{!join}</a>
|
||||
* This plugin doesn't have own name, and is called by specifying local parameter <code>{!join score=...}...</code>.
|
||||
* Note: this parser is invoked even if you specify <code>score=none</code>.
|
||||
* <br>Example:<code>q={!join from=manu_id_s to=id score=total}foo</code>
|
||||
* <ul>
|
||||
* <li>from - "foreign key" field name to collect values while enumerating subordinate query (denoted as <code>foo</code> in example above).
|
||||
* it's better to have this field declared as <code>type="string" docValues="true"</code>.
|
||||
* note: if <a href="http://wiki.apache.org/solr/DocValues">docValues</a> are not enabled for this field, it will work anyway,
|
||||
* but it costs some memory for {@link UninvertingReader}.
|
||||
* Also, numeric doc values are not supported until <a href="https://issues.apache.org/jira/browse/LUCENE-5868">LUCENE-5868</a>.
|
||||
* Thus, it only supports {@link DocValuesType#SORTED}, {@link DocValuesType#SORTED_SET}, {@link DocValuesType#BINARY}. </li>
|
||||
* <li>fromIndex - optional parameter, a core name where subordinate query should run (and <code>from</code> values are collected) rather than current core.
|
||||
* <br>Example:<code>q={!join from=manu_id_s to=id score=total fromIndex=products}foo</code>
|
||||
* <br>Follow up <a href="https://issues.apache.org/jira/browse/SOLR-7775">SOLR-7775</a> for SolrCloud collections support.</li>
|
||||
* <li>to - "primary key" field name which is searched for values collected from subordinate query.
|
||||
* it should be declared as <code>indexed="true"</code>. Now it's treated as a single value field.</li>
|
||||
* <li>score - one of {@link ScoreMode}: None,Avg,Total,Max. Lowercase is also accepted.</li>
|
||||
* </ul>
|
||||
*/
|
||||
public class ScoreJoinQParserPlugin extends QParserPlugin {
|
||||
|
||||
public static final String SCORE = "score";
|
||||
|
||||
static class OtherCoreJoinQuery extends SameCoreJoinQuery {
|
||||
private final String fromIndex;
|
||||
private final long fromCoreOpenTime;
|
||||
|
||||
public OtherCoreJoinQuery(Query fromQuery, String fromField,
|
||||
String fromIndex, long fromCoreOpenTime, ScoreMode scoreMode,
|
||||
String toField) {
|
||||
super(fromQuery, fromField, toField, scoreMode);
|
||||
this.fromIndex = fromIndex;
|
||||
this.fromCoreOpenTime = fromCoreOpenTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query rewrite(IndexReader reader) throws IOException {
|
||||
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
|
||||
|
||||
CoreContainer container = info.getReq().getCore().getCoreDescriptor().getCoreContainer();
|
||||
|
||||
final SolrCore fromCore = container.getCore(fromIndex);
|
||||
|
||||
if (fromCore == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex);
|
||||
}
|
||||
RefCounted<SolrIndexSearcher> fromHolder = null;
|
||||
fromHolder = fromCore.getRegisteredSearcher();
|
||||
final Query joinQuery;
|
||||
try {
|
||||
joinQuery = JoinUtil.createJoinQuery(fromField, true,
|
||||
toField, fromQuery, fromHolder.get(), scoreMode);
|
||||
} finally {
|
||||
fromCore.close();
|
||||
fromHolder.decref();
|
||||
}
|
||||
joinQuery.setBoost(getBoost());
|
||||
return joinQuery.rewrite(reader);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = super.hashCode();
|
||||
result = prime * result
|
||||
+ (int) (fromCoreOpenTime ^ (fromCoreOpenTime >>> 32));
|
||||
result = prime * result
|
||||
+ ((fromIndex == null) ? 0 : fromIndex.hashCode());
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (!super.equals(obj)) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
OtherCoreJoinQuery other = (OtherCoreJoinQuery) obj;
|
||||
if (fromCoreOpenTime != other.fromCoreOpenTime) return false;
|
||||
if (fromIndex == null) {
|
||||
if (other.fromIndex != null) return false;
|
||||
} else if (!fromIndex.equals(other.fromIndex)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString(String field) {
|
||||
return "OtherCoreJoinQuery [fromIndex=" + fromIndex
|
||||
+ ", fromCoreOpenTime=" + fromCoreOpenTime + " extends "
|
||||
+ super.toString(field) + "]";
|
||||
}
|
||||
}
|
||||
|
||||
static class SameCoreJoinQuery extends Query {
|
||||
protected final Query fromQuery;
|
||||
protected final ScoreMode scoreMode;
|
||||
protected final String fromField;
|
||||
protected final String toField;
|
||||
|
||||
SameCoreJoinQuery(Query fromQuery, String fromField, String toField,
|
||||
ScoreMode scoreMode) {
|
||||
this.fromQuery = fromQuery;
|
||||
this.scoreMode = scoreMode;
|
||||
this.fromField = fromField;
|
||||
this.toField = toField;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query rewrite(IndexReader reader) throws IOException {
|
||||
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
|
||||
final Query jq = JoinUtil.createJoinQuery(fromField, true,
|
||||
toField, fromQuery, info.getReq().getSearcher(), scoreMode);
|
||||
jq.setBoost(getBoost());
|
||||
return jq.rewrite(reader);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString(String field) {
|
||||
return "SameCoreJoinQuery [fromQuery=" + fromQuery + ", fromField="
|
||||
+ fromField + ", toField=" + toField + ", scoreMode=" + scoreMode
|
||||
+ "]";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = super.hashCode();
|
||||
result = prime * result
|
||||
+ ((fromField == null) ? 0 : fromField.hashCode());
|
||||
result = prime * result
|
||||
+ ((fromQuery == null) ? 0 : fromQuery.hashCode());
|
||||
result = prime * result
|
||||
+ ((scoreMode == null) ? 0 : scoreMode.hashCode());
|
||||
result = prime * result + ((toField == null) ? 0 : toField.hashCode());
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (!super.equals(obj)) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
SameCoreJoinQuery other = (SameCoreJoinQuery) obj;
|
||||
if (fromField == null) {
|
||||
if (other.fromField != null) return false;
|
||||
} else if (!fromField.equals(other.fromField)) return false;
|
||||
if (fromQuery == null) {
|
||||
if (other.fromQuery != null) return false;
|
||||
} else if (!fromQuery.equals(other.fromQuery)) return false;
|
||||
if (scoreMode != other.scoreMode) return false;
|
||||
if (toField == null) {
|
||||
if (other.toField != null) return false;
|
||||
} else if (!toField.equals(other.toField)) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
final static Map<String, ScoreMode> lowercase = Collections.unmodifiableMap( new HashMap<String, ScoreMode>() {
|
||||
{
|
||||
for (ScoreMode s : ScoreMode.values()) {
|
||||
put(s.name().toLowerCase(Locale.ROOT), s);
|
||||
put(s.name(), s);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@Override
|
||||
public void init(NamedList args) {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
return new QParser(qstr, localParams, params, req) {
|
||||
@Override
|
||||
public Query parse() throws SyntaxError {
|
||||
final String fromField = localParams.get("from");
|
||||
final String fromIndex = localParams.get("fromIndex");
|
||||
final String toField = localParams.get("to");
|
||||
final ScoreMode scoreMode = parseScore();
|
||||
|
||||
final String v = localParams.get(CommonParams.VALUE);
|
||||
|
||||
final Query q = createQuery(fromField, v, fromIndex, toField, scoreMode,
|
||||
CommonParams.TRUE.equals(localParams.get("TESTenforceSameCoreAsAnotherOne")));
|
||||
|
||||
return q;
|
||||
}
|
||||
|
||||
private Query createQuery(final String fromField, final String fromQueryStr,
|
||||
String fromIndex, final String toField, final ScoreMode scoreMode,
|
||||
boolean byPassShortCircutCheck) throws SyntaxError {
|
||||
|
||||
final String myCore = req.getCore().getCoreDescriptor().getName();
|
||||
|
||||
if (fromIndex != null && (!fromIndex.equals(myCore) || byPassShortCircutCheck)) {
|
||||
CoreContainer container = req.getCore().getCoreDescriptor().getCoreContainer();
|
||||
|
||||
final SolrCore fromCore = container.getCore(fromIndex);
|
||||
RefCounted<SolrIndexSearcher> fromHolder = null;
|
||||
|
||||
if (fromCore == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex);
|
||||
}
|
||||
|
||||
long fromCoreOpenTime = 0;
|
||||
LocalSolrQueryRequest otherReq = new LocalSolrQueryRequest(fromCore, params);
|
||||
|
||||
try {
|
||||
QParser fromQueryParser = QParser.getParser(fromQueryStr, "lucene", otherReq);
|
||||
Query fromQuery = fromQueryParser.getQuery();
|
||||
|
||||
fromHolder = fromCore.getRegisteredSearcher();
|
||||
if (fromHolder != null) {
|
||||
fromCoreOpenTime = fromHolder.get().getOpenTime();
|
||||
}
|
||||
return new OtherCoreJoinQuery(fromQuery, fromField, fromIndex, fromCoreOpenTime,
|
||||
scoreMode, toField);
|
||||
} finally {
|
||||
otherReq.close();
|
||||
fromCore.close();
|
||||
if (fromHolder != null) fromHolder.decref();
|
||||
}
|
||||
} else {
|
||||
QParser fromQueryParser = subQuery(fromQueryStr, null);
|
||||
final Query fromQuery = fromQueryParser.getQuery();
|
||||
return new SameCoreJoinQuery(fromQuery, fromField, toField, scoreMode);
|
||||
}
|
||||
}
|
||||
|
||||
private ScoreMode parseScore() {
|
||||
|
||||
String score = getParam(SCORE);
|
||||
final ScoreMode scoreMode = lowercase.get(score);
|
||||
if (scoreMode == null) {
|
||||
throw new IllegalArgumentException("Unable to parse ScoreMode from: " + score);
|
||||
}
|
||||
return scoreMode;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
<?xml version="1.0" ?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<schema name="doc-values-for-Join" version="1.5">
|
||||
<types>
|
||||
<fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
|
||||
<fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
|
||||
<fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
|
||||
<fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
|
||||
<fieldType name="date" class="solr.TrieDateField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
|
||||
<fieldtype name="string" class="solr.StrField" sortMissingLast="true"/>
|
||||
<fieldType name="text" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true" >
|
||||
<analyzer type="index">
|
||||
<tokenizer class="solr.MockTokenizerFactory"/>
|
||||
<filter class="solr.StopFilterFactory"
|
||||
ignoreCase="true"
|
||||
words="stopwords.txt"
|
||||
/>
|
||||
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
|
||||
<filter class="solr.LowerCaseFilterFactory"/>
|
||||
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
|
||||
<filter class="solr.PorterStemFilterFactory"/>
|
||||
</analyzer>
|
||||
<analyzer type="query">
|
||||
<tokenizer class="solr.MockTokenizerFactory"/>
|
||||
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
|
||||
<filter class="solr.StopFilterFactory"
|
||||
ignoreCase="true"
|
||||
words="stopwords.txt"
|
||||
/>
|
||||
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
|
||||
<filter class="solr.LowerCaseFilterFactory"/>
|
||||
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
|
||||
<filter class="solr.PorterStemFilterFactory"/>
|
||||
</analyzer>
|
||||
</fieldType>
|
||||
</types>
|
||||
|
||||
<fields>
|
||||
<field name="id" type="string" indexed="true" stored="true" docValues="false" multiValued="false" required="true"/>
|
||||
<field name="id_dv" type="string" indexed="false" stored="false" docValues="true" multiValued="false" required="true"/>
|
||||
<dynamicField name="*_i" type="int" indexed="true" stored="false" docValues="false"/>
|
||||
<dynamicField name="*_i_dv" type="int" indexed="true" stored="true" docValues="true"/>
|
||||
<dynamicField name="*_is" type="int" indexed="true" stored="false" docValues="false" multiValued="true"/>
|
||||
<dynamicField name="*_is_dv" type="int" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_s" type="string" indexed="true" stored="false" docValues="false"/>
|
||||
<dynamicField name="*_s_dv" type="string" indexed="true" stored="true" docValues="true"/>
|
||||
<dynamicField name="*_ss" type="string" indexed="true" stored="false" docValues="false" multiValued="true"/>
|
||||
<dynamicField name="*_ss_dv" type="string" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_f" type="float" indexed="true" stored="false" docValues="false"/>
|
||||
<dynamicField name="*_f_dv" type="float" indexed="true" stored="true" docValues="true"/>
|
||||
<dynamicField name="*_fs_dv" type="float" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_l" type="long" indexed="true" stored="false" docValues="false"/>
|
||||
<dynamicField name="*_l_dv" type="long" indexed="true" stored="false" docValues="true"/>
|
||||
<dynamicField name="*_ls_dv" type="long" indexed="true" stored="false" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_d" type="double" indexed="true" stored="false" docValues="false"/>
|
||||
<dynamicField name="*_d_dv" type="double" indexed="true" stored="false" docValues="true"/>
|
||||
<dynamicField name="*_ds_dv" type="double" indexed="true" stored="false" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_dt" type="date" indexed="true" stored="false" docValues="false"/>
|
||||
<dynamicField name="*_dt_dv" type="date" indexed="true" stored="false" docValues="true"/>
|
||||
<dynamicField name="*_dts_dv" type="date" indexed="true" stored="false" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
|
||||
</fields>
|
||||
|
||||
<defaultSearchField>id</defaultSearchField>
|
||||
<uniqueKey>id</uniqueKey>
|
||||
|
||||
<copyField source="*_i" dest="*_i_dv" />
|
||||
<copyField source="*_f" dest="*_f_dv" />
|
||||
<copyField source="*_is" dest="*_is_dv" />
|
||||
<copyField source="*_s" dest="*_s_dv" />
|
||||
<copyField source="*_ss" dest="*_ss_dv" />
|
||||
<copyField source="id" dest="id_dv" />
|
||||
</schema>
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr;
|
||||
|
||||
import java.io.StringWriter;
|
||||
import java.util.Collections;
|
||||
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.core.CoreDescriptor;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestHandler;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.response.QueryResponseWriter;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.search.join.TestScoreJoinQPNoScore;
|
||||
import org.apache.solr.servlet.DirectSolrConnection;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestCrossCoreJoin extends SolrTestCaseJ4 {
|
||||
|
||||
private static SolrCore fromCore;
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeTests() throws Exception {
|
||||
System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
|
||||
// initCore("solrconfig.xml","schema12.xml");
|
||||
|
||||
// File testHome = createTempDir().toFile();
|
||||
// FileUtils.copyDirectory(getFile("solrj/solr"), testHome);
|
||||
initCore("solrconfig.xml", "schema12.xml", TEST_HOME(), "collection1");
|
||||
final CoreContainer coreContainer = h.getCoreContainer();
|
||||
final CoreDescriptor toCoreDescriptor = coreContainer.getCoreDescriptor("collection1");
|
||||
final CoreDescriptor fromCoreDescriptor = new CoreDescriptor("fromCore", toCoreDescriptor) {
|
||||
@Override
|
||||
public String getSchemaName() {
|
||||
return "schema.xml";
|
||||
}
|
||||
};
|
||||
|
||||
fromCore = coreContainer.create(fromCoreDescriptor);
|
||||
|
||||
assertU(add(doc("id", "1", "name", "john", "title", "Director", "dept_s", "Engineering")));
|
||||
assertU(add(doc("id", "2", "name", "mark", "title", "VP", "dept_s", "Marketing")));
|
||||
assertU(add(doc("id", "3", "name", "nancy", "title", "MTS", "dept_s", "Sales")));
|
||||
assertU(add(doc("id", "4", "name", "dave", "title", "MTS", "dept_s", "Support", "dept_s", "Engineering")));
|
||||
assertU(add(doc("id", "5", "name", "tina", "title", "VP", "dept_s", "Engineering")));
|
||||
assertU(commit());
|
||||
|
||||
update(fromCore, add(doc("id", "10", "dept_id_s", "Engineering", "text", "These guys develop stuff", "cat", "dev")));
|
||||
update(fromCore, add(doc("id", "11", "dept_id_s", "Marketing", "text", "These guys make you look good")));
|
||||
update(fromCore, add(doc("id", "12", "dept_id_s", "Sales", "text", "These guys sell stuff")));
|
||||
update(fromCore, add(doc("id", "13", "dept_id_s", "Support", "text", "These guys help customers")));
|
||||
update(fromCore, commit());
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static String update(SolrCore core, String xml) throws Exception {
|
||||
DirectSolrConnection connection = new DirectSolrConnection(core);
|
||||
SolrRequestHandler handler = core.getRequestHandler("/update");
|
||||
return connection.request(handler, null, xml);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJoin() throws Exception {
|
||||
doTestJoin("{!join");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testScoreJoin() throws Exception {
|
||||
doTestJoin("{!join " + TestScoreJoinQPNoScore.whateverScore());
|
||||
}
|
||||
|
||||
void doTestJoin(String joinPrefix) throws Exception {
|
||||
assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id",
|
||||
"debugQuery", random().nextBoolean() ? "true":"false")
|
||||
, "/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
|
||||
);
|
||||
|
||||
// find people that develop stuff - but limit via filter query to a name of "john"
|
||||
// this tests filters being pushed down to queries (SOLR-3062)
|
||||
assertJQ(req("q", joinPrefix + " from=dept_id_s to=dept_s fromIndex=fromCore}cat:dev", "fl", "id", "fq", "name:john",
|
||||
"debugQuery", random().nextBoolean() ? "true":"false")
|
||||
, "/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}"
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCoresAreDifferent() throws Exception {
|
||||
assertQEx("schema12.xml" + " has no \"cat\" field", req("cat:*"), ErrorCode.BAD_REQUEST);
|
||||
final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, "cat:*", "lucene", 0, 100, Collections.emptyMap());
|
||||
final String resp = query(fromCore, req);
|
||||
assertTrue(resp, resp.contains("numFound=\"1\""));
|
||||
assertTrue(resp, resp.contains("<int name=\"id\">10</int>"));
|
||||
|
||||
}
|
||||
|
||||
public String query(SolrCore core, SolrQueryRequest req) throws Exception {
|
||||
String handler = "standard";
|
||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
||||
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
|
||||
core.execute(core.getRequestHandler(handler), req, rsp);
|
||||
if (rsp.getException() != null) {
|
||||
throw rsp.getException();
|
||||
}
|
||||
StringWriter sw = new StringWriter(32000);
|
||||
QueryResponseWriter responseWriter = core.getQueryResponseWriter(req);
|
||||
responseWriter.write(sw, req, rsp);
|
||||
req.close();
|
||||
SolrRequestInfo.clearRequestInfo();
|
||||
return sw.toString();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void nukeAll() {
|
||||
fromCore = null;
|
||||
}
|
||||
}
|
||||
|
|
@ -384,6 +384,23 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
|
|||
}
|
||||
}
|
||||
|
||||
public void testQueryScoreJoin() throws Exception {
|
||||
SolrQueryRequest req = req("myVar", "5",
|
||||
"df", "text",
|
||||
"ff", "foo_s",
|
||||
"tt", "bar_s",
|
||||
"scoreavg","avg");
|
||||
|
||||
try {
|
||||
assertQueryEquals("join", req,
|
||||
"{!join from=foo_s to=bar_s score=avg}asdf",
|
||||
"{!join from=$ff to=$tt score=Avg}asdf",
|
||||
"{!join from=$ff to='bar_s' score=$scoreavg}text:asdf");
|
||||
} finally {
|
||||
req.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void testTerms() throws Exception {
|
||||
assertQueryEquals("terms", "{!terms f=foo_i}10,20,30,-10,-20,-30", "{!terms f=foo_i}10,20,30,-10,-20,-30");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,357 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.join.ScoreMode;
|
||||
import org.apache.solr.JSONTestUtil;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.params.MapSolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.search.JoinQParserPlugin;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.noggit.JSONUtil;
|
||||
import org.noggit.ObjectBuilder;
|
||||
|
||||
public class TestScoreJoinQPNoScore extends SolrTestCaseJ4 {
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeTests() throws Exception {
|
||||
System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
|
||||
initCore("solrconfig-basic.xml","schema-docValuesJoin.xml");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJoin() throws Exception {
|
||||
assertU(add(doc("id", "1","name_s", "john", "title_s", "Director", "dept_ss","Engineering")));
|
||||
assertU(add(doc("id", "2","name_s", "mark", "title_s", "VP", "dept_ss","Marketing")));
|
||||
assertU(add(doc("id", "3","name_s", "nancy", "title_s", "MTS", "dept_ss","Sales")));
|
||||
assertU(add(doc("id", "4","name_s", "dave", "title_s", "MTS", "dept_ss","Support", "dept_ss","Engineering")));
|
||||
assertU(add(doc("id", "5","name_s", "tina", "title_s", "VP", "dept_ss","Engineering")));
|
||||
|
||||
assertU(add(doc("id","10", "dept_id_s", "Engineering", "text_t","These guys develop stuff")));
|
||||
assertU(add(doc("id","11", "dept_id_s", "Marketing", "text_t","These guys make you look good")));
|
||||
assertU(add(doc("id","12", "dept_id_s", "Sales", "text_t","These guys sell stuff")));
|
||||
assertU(add(doc("id","13", "dept_id_s", "Support", "text_t","These guys help customers")));
|
||||
|
||||
assertU(commit());
|
||||
|
||||
// test debugging TODO no debug in JoinUtil
|
||||
// assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id", "debugQuery","true")
|
||||
// ,"/debug/join/{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS=={'_MATCH_':'fromSetSize,toSetSize', 'fromSetSize':2, 'toSetSize':3}"
|
||||
// );
|
||||
|
||||
assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id")
|
||||
,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}"
|
||||
);
|
||||
|
||||
// empty from
|
||||
assertJQ(req("q","{!join from=noexist_s to=dept_id_s"+whateverScore()+"}*:*", "fl","id")
|
||||
,"/response=={'numFound':0,'start':0,'docs':[]}"
|
||||
);
|
||||
|
||||
// empty to
|
||||
assertJQ(req("q","{!join from=dept_ss to=noexist_s"+whateverScore()+"}*:*", "fl","id")
|
||||
,"/response=={'numFound':0,'start':0,'docs':[]}"
|
||||
);
|
||||
|
||||
// self join... return everyone with she same title as Dave
|
||||
assertJQ(req("q","{!join from=title_s to=title_s"+whateverScore()+"}name_s:dave", "fl","id")
|
||||
,"/response=={'numFound':2,'start':0,'docs':[{'id':'3'},{'id':'4'}]}"
|
||||
);
|
||||
|
||||
// find people that develop stuff
|
||||
assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id")
|
||||
,"/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
|
||||
);
|
||||
|
||||
// self join on multivalued text_t field
|
||||
assertJQ(req("q","{!join from=title_s to=title_s"+whateverScore()+"}name_s:dave", "fl","id")
|
||||
,"/response=={'numFound':2,'start':0,'docs':[{'id':'3'},{'id':'4'}]}"
|
||||
);
|
||||
|
||||
assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id", "debugQuery","true")
|
||||
,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}"
|
||||
);
|
||||
|
||||
// expected outcome for a sub query matching dave joined against departments
|
||||
final String davesDepartments =
|
||||
"/response=={'numFound':2,'start':0,'docs':[{'id':'10'},{'id':'13'}]}";
|
||||
|
||||
// straight forward query
|
||||
assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}name_s:dave",
|
||||
"fl","id"),
|
||||
davesDepartments);
|
||||
|
||||
// variable deref for sub-query parsing
|
||||
assertJQ(req("q","{!join from=dept_ss to=dept_id_s v=$qq"+whateverScore()+"}",
|
||||
"qq","{!dismax}dave",
|
||||
"qf","name_s",
|
||||
"fl","id",
|
||||
"debugQuery","true"),
|
||||
davesDepartments);
|
||||
|
||||
// variable deref for sub-query parsing w/localparams
|
||||
assertJQ(req("q","{!join from=dept_ss to=dept_id_s v=$qq"+whateverScore()+"}",
|
||||
"qq","{!dismax qf=name_s}dave",
|
||||
"fl","id",
|
||||
"debugQuery","true"),
|
||||
davesDepartments);
|
||||
|
||||
// defType local param to control sub-query parsing
|
||||
assertJQ(req("q","{!join from=dept_ss to=dept_id_s defType=dismax"+whateverScore()+"}dave",
|
||||
"qf","name_s",
|
||||
"fl","id",
|
||||
"debugQuery","true"),
|
||||
davesDepartments);
|
||||
|
||||
// find people that develop stuff - but limit via filter query to a name of "john"
|
||||
// this tests filters being pushed down to queries (SOLR-3062)
|
||||
assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id", "fq", "name_s:john")
|
||||
,"/response=={'numFound':1,'start':0,'docs':[{'id':'1'}]}"
|
||||
);
|
||||
|
||||
|
||||
assertJQ(req("q","{!join from=dept_ss to=dept_id_s"+whateverScore()+"}title_s:MTS", "fl","id"
|
||||
)
|
||||
,"/response=={'numFound':3,'start':0,'docs':[{'id':'10'},{'id':'12'},{'id':'13'}]}");
|
||||
|
||||
// find people that develop stuff, even if it's requested as single value
|
||||
assertJQ(req("q","{!join from=dept_id_s to=dept_ss"+whateverScore()+"}text_t:develop", "fl","id")
|
||||
,"/response=={'numFound':3,'start':0,'docs':[{'id':'1'},{'id':'4'},{'id':'5'}]}"
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
public void testJoinQueryType() throws SyntaxError, IOException{
|
||||
SolrQueryRequest req = null;
|
||||
try{
|
||||
final String score = whateverScore();
|
||||
|
||||
req = req("{!join from=dept_id_s to=dept_ss"+score+"}text_t:develop");
|
||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
||||
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
|
||||
|
||||
{
|
||||
final Query query = QParser.getParser(req.getParams().get("q"), null, req).getQuery();
|
||||
final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader());
|
||||
assertTrue(
|
||||
rewrittenQuery+" should be Lucene's",
|
||||
rewrittenQuery.getClass().getPackage().getName()
|
||||
.startsWith("org.apache.lucene"));
|
||||
}
|
||||
{
|
||||
final Query query = QParser.getParser(
|
||||
"{!join from=dept_id_s to=dept_ss}text_t:develop"
|
||||
, null, req).getQuery();
|
||||
final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader());
|
||||
assertEquals(rewrittenQuery+" is expected to be from Solr",
|
||||
JoinQParserPlugin.class.getPackage().getName(),
|
||||
rewrittenQuery.getClass().getPackage().getName());
|
||||
}
|
||||
}finally{
|
||||
if(req!=null){
|
||||
req.close();
|
||||
}
|
||||
SolrRequestInfo.clearRequestInfo();
|
||||
}
|
||||
}
|
||||
|
||||
public static String whateverScore() {
|
||||
final ScoreMode[] vals = ScoreMode.values();
|
||||
return " score="+vals[random().nextInt(vals.length)]+" ";
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRandomJoin() throws Exception {
|
||||
int indexIter=50 * RANDOM_MULTIPLIER;
|
||||
int queryIter=50 * RANDOM_MULTIPLIER;
|
||||
|
||||
// groups of fields that have any chance of matching... used to
|
||||
// increase test effectiveness by avoiding 0 resultsets much of the time.
|
||||
String[][] compat = new String[][] {
|
||||
{"small_s_dv","small2_s_dv","small2_ss_dv","small3_ss_dv"},
|
||||
{"small_i_dv","small2_i_dv","small2_is_dv","small3_is_dv"}
|
||||
};
|
||||
|
||||
|
||||
while (--indexIter >= 0) {
|
||||
int indexSize = random().nextInt(20 * RANDOM_MULTIPLIER);
|
||||
|
||||
List<FldType> types = new ArrayList<FldType>();
|
||||
types.add(new FldType("id",ONE_ONE, new SVal('A','Z',4,4)));
|
||||
/** no numeric fields so far LUCENE-5868
|
||||
types.add(new FldType("score_f_dv",ONE_ONE, new FVal(1,100))); // field used to score
|
||||
**/
|
||||
types.add(new FldType("small_s_dv",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
|
||||
types.add(new FldType("small2_s_dv",ZERO_ONE, new SVal('a',(char)('c'+indexSize/3),1,1)));
|
||||
types.add(new FldType("small2_ss_dv",ZERO_TWO, new SVal('a',(char)('c'+indexSize/3),1,1)));
|
||||
types.add(new FldType("small3_ss_dv",new IRange(0,25), new SVal('A','z',1,1)));
|
||||
/** no numeric fields so far LUCENE-5868
|
||||
types.add(new FldType("small_i_dv",ZERO_ONE, new IRange(0,5+indexSize/3)));
|
||||
types.add(new FldType("small2_i_dv",ZERO_ONE, new IRange(0,5+indexSize/3)));
|
||||
types.add(new FldType("small2_is_dv",ZERO_TWO, new IRange(0,5+indexSize/3)));
|
||||
types.add(new FldType("small3_is_dv",new IRange(0,25), new IRange(0,100)));
|
||||
**/
|
||||
|
||||
clearIndex();
|
||||
Map<Comparable, Doc> model = indexDocs(types, null, indexSize);
|
||||
Map<String, Map<Comparable, Set<Comparable>>> pivots = new HashMap<String, Map<Comparable, Set<Comparable>>>();
|
||||
|
||||
for (int qiter=0; qiter<queryIter; qiter++) {
|
||||
String fromField;
|
||||
String toField;
|
||||
if (random().nextInt(100) < 5) {
|
||||
// pick random fields 5% of the time
|
||||
fromField = types.get(random().nextInt(types.size())).fname;
|
||||
// pick the same field 50% of the time we pick a random field (since other fields won't match anything)
|
||||
toField = (random().nextInt(100) < 50) ? fromField : types.get(random().nextInt(types.size())).fname;
|
||||
} else {
|
||||
// otherwise, pick compatible fields that have a chance of matching indexed tokens
|
||||
String[] group = compat[random().nextInt(compat.length)];
|
||||
fromField = group[random().nextInt(group.length)];
|
||||
toField = group[random().nextInt(group.length)];
|
||||
}
|
||||
|
||||
Map<Comparable, Set<Comparable>> pivot = pivots.get(fromField+"/"+toField);
|
||||
if (pivot == null) {
|
||||
pivot = createJoinMap(model, fromField, toField);
|
||||
pivots.put(fromField+"/"+toField, pivot);
|
||||
}
|
||||
|
||||
Collection<Doc> fromDocs = model.values();
|
||||
Set<Comparable> docs = join(fromDocs, pivot);
|
||||
List<Doc> docList = new ArrayList<Doc>(docs.size());
|
||||
for (Comparable id : docs) docList.add(model.get(id));
|
||||
Collections.sort(docList, createComparator("_docid_",true,false,false,false));
|
||||
List sortedDocs = new ArrayList();
|
||||
for (Doc doc : docList) {
|
||||
if (sortedDocs.size() >= 10) break;
|
||||
sortedDocs.add(doc.toObject(h.getCore().getLatestSchema()));
|
||||
}
|
||||
|
||||
Map<String,Object> resultSet = new LinkedHashMap<String,Object>();
|
||||
resultSet.put("numFound", docList.size());
|
||||
resultSet.put("start", 0);
|
||||
resultSet.put("docs", sortedDocs);
|
||||
|
||||
// todo: use different join queries for better coverage
|
||||
|
||||
SolrQueryRequest req = req("wt","json","indent","true", "echoParams","all",
|
||||
"q","{!join from="+fromField+" to="+toField
|
||||
+" "+ (random().nextBoolean() ? "fromIndex=collection1" : "")
|
||||
+" "+ (random().nextBoolean() ? "TESTenforceSameCoreAsAnotherOne=true" : "")
|
||||
+" "+whateverScore()+"}*:*"
|
||||
, "sort", "_docid_ asc"
|
||||
);
|
||||
|
||||
String strResponse = h.query(req);
|
||||
|
||||
Object realResponse = ObjectBuilder.fromJSON(strResponse);
|
||||
String err = JSONTestUtil.matchObj("/response", realResponse, resultSet);
|
||||
if (err != null) {
|
||||
final String m = "JOIN MISMATCH: " + err
|
||||
+ "\n\trequest="+req
|
||||
+ "\n\tresult="+strResponse
|
||||
+ "\n\texpected="+ JSONUtil.toJSON(resultSet)
|
||||
;// + "\n\tmodel="+ JSONUtil.toJSON(model);
|
||||
log.error(m);
|
||||
{
|
||||
SolrQueryRequest f = req("wt","json","indent","true", "echoParams","all",
|
||||
"q","*:*", "facet","true",
|
||||
"facet.field", fromField
|
||||
, "sort", "_docid_ asc"
|
||||
,"rows","0"
|
||||
);
|
||||
log.error("faceting on from field: "+h.query(f));
|
||||
}
|
||||
{
|
||||
final Map<String,String> ps = ((MapSolrParams)req.getParams()).getMap();
|
||||
final String q = ps.get("q");
|
||||
ps.put("q", q.replaceAll("join score=none", "join"));
|
||||
log.error("plain join: "+h.query(req));
|
||||
ps.put("q", q);
|
||||
|
||||
}
|
||||
{
|
||||
// re-execute the request... good for putting a breakpoint here for debugging
|
||||
final Map<String,String> ps = ((MapSolrParams)req.getParams()).getMap();
|
||||
final String q = ps.get("q");
|
||||
ps.put("q", q.replaceAll("\\}", " cache=false\\}"));
|
||||
String rsp = h.query(req);
|
||||
}
|
||||
fail(err);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Map<Comparable, Set<Comparable>> createJoinMap(Map<Comparable, Doc> model, String fromField, String toField) {
|
||||
Map<Comparable, Set<Comparable>> id_to_id = new HashMap<Comparable, Set<Comparable>>();
|
||||
|
||||
Map<Comparable, List<Comparable>> value_to_id = invertField(model, toField);
|
||||
|
||||
for (Comparable fromId : model.keySet()) {
|
||||
Doc doc = model.get(fromId);
|
||||
List<Comparable> vals = doc.getValues(fromField);
|
||||
if (vals == null) continue;
|
||||
for (Comparable val : vals) {
|
||||
List<Comparable> toIds = value_to_id.get(val);
|
||||
if (toIds == null) continue;
|
||||
Set<Comparable> ids = id_to_id.get(fromId);
|
||||
if (ids == null) {
|
||||
ids = new HashSet<Comparable>();
|
||||
id_to_id.put(fromId, ids);
|
||||
}
|
||||
for (Comparable toId : toIds)
|
||||
ids.add(toId);
|
||||
}
|
||||
}
|
||||
|
||||
return id_to_id;
|
||||
}
|
||||
|
||||
|
||||
Set<Comparable> join(Collection<Doc> input, Map<Comparable, Set<Comparable>> joinMap) {
|
||||
Set<Comparable> ids = new HashSet<Comparable>();
|
||||
for (Doc doc : input) {
|
||||
Collection<Comparable> output = joinMap.get(doc.id);
|
||||
if (output == null) continue;
|
||||
ids.addAll(output);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,330 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.join.ScoreMode;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.SolrCache;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
|
||||
public class TestScoreJoinQPScore extends SolrTestCaseJ4 {
|
||||
|
||||
private static final String idField = "id";
|
||||
private static final String toField = "movieId_s";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeTests() throws Exception {
|
||||
System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
|
||||
initCore("solrconfig.xml", "schema12.xml");
|
||||
}
|
||||
|
||||
public void testSimple() throws Exception {
|
||||
final String idField = "id";
|
||||
final String toField = "productId_s";
|
||||
|
||||
clearIndex();
|
||||
|
||||
// 0
|
||||
assertU(add(doc("t_description", "random text",
|
||||
"name", "name1",
|
||||
idField, "1")));
|
||||
|
||||
// 1
|
||||
|
||||
assertU(add(doc("price_s", "10.0",
|
||||
idField, "2",
|
||||
toField, "1")));
|
||||
// 2
|
||||
assertU(add(doc("price_s", "20.0",
|
||||
idField, "3",
|
||||
toField, "1")));
|
||||
// 3
|
||||
assertU(add(doc("t_description", "more random text",
|
||||
"name", "name2",
|
||||
idField, "4")));
|
||||
// 4
|
||||
assertU(add(doc("price_s", "10.0",
|
||||
idField, "5",
|
||||
toField, "4")));
|
||||
// 5
|
||||
assertU(add(doc("price_s", "20.0",
|
||||
idField, "6",
|
||||
toField, "4")));
|
||||
|
||||
assertU(commit());
|
||||
|
||||
// Search for product
|
||||
assertJQ(req("q", "{!join from=" + idField + " to=" + toField + " score=None}name:name2", "fl", "id")
|
||||
, "/response=={'numFound':2,'start':0,'docs':[{'id':'5'},{'id':'6'}]}");
|
||||
|
||||
/*Query joinQuery =
|
||||
JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name2")), indexSearcher, ScoreMode.None);
|
||||
|
||||
TopDocs result = indexSearcher.search(joinQuery, 10);
|
||||
assertEquals(2, result.totalHits);
|
||||
assertEquals(4, result.scoreDocs[0].doc);
|
||||
assertEquals(5, result.scoreDocs[1].doc);
|
||||
*/
|
||||
assertJQ(req("q", "{!join from=" + idField + " to=" + toField + " score=None}name:name1", "fl", "id")
|
||||
, "/response=={'numFound':2,'start':0,'docs':[{'id':'2'},{'id':'3'}]}");
|
||||
|
||||
/*joinQuery = JoinUtil.createJoinQuery(idField, false, toField, new TermQuery(new Term("name", "name1")), indexSearcher, ScoreMode.None);
|
||||
result = indexSearcher.search(joinQuery, 10);
|
||||
assertEquals(2, result.totalHits);
|
||||
assertEquals(1, result.scoreDocs[0].doc);
|
||||
assertEquals(2, result.scoreDocs[1].doc);*/
|
||||
|
||||
// Search for offer
|
||||
assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=None}id:5", "fl", "id")
|
||||
, "/response=={'numFound':1,'start':0,'docs':[{'id':'4'}]}");
|
||||
/*joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("id", "5")), indexSearcher, ScoreMode.None);
|
||||
result = indexSearcher.search(joinQuery, 10);
|
||||
assertEquals(1, result.totalHits);
|
||||
assertEquals(3, result.scoreDocs[0].doc);
|
||||
|
||||
indexSearcher.getIndexReader().close();
|
||||
dir.close();*/
|
||||
}
|
||||
|
||||
public void testSimpleWithScoring() throws Exception {
|
||||
indexDataForScorring();
|
||||
|
||||
// Search for movie via subtitle
|
||||
assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Max}title:random", "fl", "id")
|
||||
, "/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'4'}]}");
|
||||
//dump(req("q","{!scorejoin from="+toField+" to="+idField+" score=Max}title:random", "fl","id,score", "debug", "true"));
|
||||
/*
|
||||
Query joinQuery =
|
||||
JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "random")), indexSearcher, ScoreMode.Max);
|
||||
TopDocs result = indexSearcher.search(joinQuery, 10);
|
||||
assertEquals(2, result.totalHits);
|
||||
assertEquals(0, result.scoreDocs[0].doc);
|
||||
assertEquals(3, result.scoreDocs[1].doc);*/
|
||||
|
||||
|
||||
// Score mode max.
|
||||
//dump(req("q","{!scorejoin from="+toField+" to="+idField+" score=Max}title:movie", "fl","id,score", "debug", "true"));
|
||||
|
||||
// dump(req("q","title:movie", "fl","id,score", "debug", "true"));
|
||||
assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Max}title:movie", "fl", "id")
|
||||
, "/response=={'numFound':2,'start':0,'docs':[{'id':'4'},{'id':'1'}]}");
|
||||
|
||||
/*joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Max);
|
||||
result = indexSearcher.search(joinQuery, 10);
|
||||
assertEquals(2, result.totalHits);
|
||||
assertEquals(3, result.scoreDocs[0].doc);
|
||||
assertEquals(0, result.scoreDocs[1].doc);*/
|
||||
|
||||
// Score mode total
|
||||
assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Total}title:movie", "fl", "id")
|
||||
, "/response=={'numFound':2,'start':0,'docs':[{'id':'1'},{'id':'4'}]}");
|
||||
/* joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Total);
|
||||
result = indexSearcher.search(joinQuery, 10);
|
||||
assertEquals(2, result.totalHits);
|
||||
assertEquals(0, result.scoreDocs[0].doc);
|
||||
assertEquals(3, result.scoreDocs[1].doc);
|
||||
*/
|
||||
//Score mode avg
|
||||
assertJQ(req("q", "{!join from=" + toField + " to=" + idField + " score=Avg}title:movie", "fl", "id")
|
||||
, "/response=={'numFound':2,'start':0,'docs':[{'id':'4'},{'id':'1'}]}");
|
||||
|
||||
/* joinQuery = JoinUtil.createJoinQuery(toField, false, idField, new TermQuery(new Term("title", "movie")), indexSearcher, ScoreMode.Avg);
|
||||
result = indexSearcher.search(joinQuery, 10);
|
||||
assertEquals(2, result.totalHits);
|
||||
assertEquals(3, result.scoreDocs[0].doc);
|
||||
assertEquals(0, result.scoreDocs[1].doc);*/
|
||||
|
||||
}
|
||||
|
||||
final static Comparator<String> lessFloat = new Comparator<String>() {
|
||||
@Override
|
||||
public int compare(String o1, String o2) {
|
||||
assertTrue(Float.parseFloat(o1) < Float.parseFloat(o2));
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
@Ignore("SOLR-7814, also don't forget cover boost at testCacheHit()")
|
||||
public void testBoost() throws Exception {
|
||||
indexDataForScorring();
|
||||
ScoreMode score = ScoreMode.values()[random().nextInt(ScoreMode.values().length)];
|
||||
|
||||
final SolrQueryRequest req = req("q", "{!join from=movieId_s to=id score=" + score + " b=200}title:movie", "fl", "id,score", "omitHeader", "true");
|
||||
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, new SolrQueryResponse()));
|
||||
final Query luceneQ = QParser.getParser(req.getParams().get("q"), null, req).getQuery().rewrite(req.getSearcher().getLeafReader());
|
||||
assertEquals("" + luceneQ, Float.floatToIntBits(200), Float.floatToIntBits(luceneQ.getBoost()));
|
||||
SolrRequestInfo.clearRequestInfo();
|
||||
req.close();
|
||||
}
|
||||
|
||||
public void testCacheHit() throws Exception {
|
||||
indexDataForScorring();
|
||||
|
||||
SolrCache cache = (SolrCache) h.getCore().getInfoRegistry()
|
||||
.get("queryResultCache");
|
||||
{
|
||||
final NamedList statPre = cache.getStatistics();
|
||||
h.query(req("q", "{!join from=movieId_s to=id score=Avg}title:first", "fl", "id", "omitHeader", "true"));
|
||||
assertHitOrInsert(cache, statPre);
|
||||
}
|
||||
|
||||
{
|
||||
final NamedList statPre = cache.getStatistics();
|
||||
h.query(req("q", "{!join from=movieId_s to=id score=Avg}title:first", "fl", "id", "omitHeader", "true"));
|
||||
assertHit(cache, statPre);
|
||||
}
|
||||
|
||||
{
|
||||
NamedList statPre = cache.getStatistics();
|
||||
|
||||
Random r = random();
|
||||
boolean changed = false;
|
||||
boolean x = false;
|
||||
String from = (x = r.nextBoolean()) ? "id" : "movieId_s";
|
||||
changed |= x;
|
||||
String to = (x = r.nextBoolean()) ? "movieId_s" : "id";
|
||||
changed |= x;
|
||||
String score = (x = r.nextBoolean()) ? not(ScoreMode.Avg).name() : "Avg";
|
||||
changed |= x;
|
||||
/* till SOLR-7814
|
||||
* String boost = (x = r.nextBoolean()) ? "23" : "1";
|
||||
changed |= x; */
|
||||
String q = (!changed) ? (r.nextBoolean() ? "title:first^67" : "title:night") : "title:first";
|
||||
|
||||
final String resp = h.query(req("q", "{!join from=" + from + " to=" + to +
|
||||
" score=" + score +
|
||||
//" b=" + boost +
|
||||
"}" + q, "fl", "id", "omitHeader", "true")
|
||||
);
|
||||
assertInsert(cache, statPre);
|
||||
|
||||
statPre = cache.getStatistics();
|
||||
final String repeat = h.query(req("q", "{!join from=" + from + " to=" + to + " score=" + score.toLowerCase(Locale.ROOT) +
|
||||
//" b=" + boost
|
||||
"}" + q, "fl", "id", "omitHeader", "true")
|
||||
);
|
||||
assertHit(cache, statPre);
|
||||
|
||||
assertEquals("lowercase shouldn't change anything", resp, repeat);
|
||||
|
||||
try {
|
||||
h.query(req("q", "{!join from=" + from + " to=" + to + " score=" + score.substring(0, score.length() - 1) +
|
||||
"}" + q, "fl", "id", "omitHeader", "true")
|
||||
);
|
||||
fail("excpecting exception");
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertTrue(e.getMessage().contains("ScoreMode"));
|
||||
}
|
||||
}
|
||||
// this queries are not overlap, with other in this test case.
|
||||
// however it might be better to extract this method into the separate suite
|
||||
// for a while let's nuke a cache content, in case of repetitions
|
||||
cache.clear();
|
||||
}
|
||||
|
||||
private ScoreMode not(ScoreMode s) {
|
||||
Random r = random();
|
||||
final List<ScoreMode> l = new ArrayList(Arrays.asList(ScoreMode.values()));
|
||||
l.remove(s);
|
||||
return l.get(r.nextInt(l.size()));
|
||||
}
|
||||
|
||||
private void assertInsert(SolrCache cache, final NamedList statPre) {
|
||||
assertEquals("it lookups", 1,
|
||||
delta("lookups", cache.getStatistics(), statPre));
|
||||
assertEquals("it doesn't hit", 0, delta("hits", cache.getStatistics(), statPre));
|
||||
assertEquals("it inserts", 1,
|
||||
delta("inserts", cache.getStatistics(), statPre));
|
||||
}
|
||||
|
||||
private void assertHit(SolrCache cache, final NamedList statPre) {
|
||||
assertEquals("it lookups", 1,
|
||||
delta("lookups", cache.getStatistics(), statPre));
|
||||
assertEquals("it hits", 1, delta("hits", cache.getStatistics(), statPre));
|
||||
assertEquals("it doesn't insert", 0,
|
||||
delta("inserts", cache.getStatistics(), statPre));
|
||||
}
|
||||
|
||||
private void assertHitOrInsert(SolrCache cache, final NamedList statPre) {
|
||||
assertEquals("it lookups", 1,
|
||||
delta("lookups", cache.getStatistics(), statPre));
|
||||
final long mayHit = delta("hits", cache.getStatistics(), statPre);
|
||||
assertTrue("it may hit", 0 == mayHit || 1 == mayHit);
|
||||
assertEquals("or insert on cold", 1,
|
||||
delta("inserts", cache.getStatistics(), statPre) + mayHit);
|
||||
}
|
||||
|
||||
private long delta(String key, NamedList a, NamedList b) {
|
||||
return (Long) a.get(key) - (Long) b.get(key);
|
||||
}
|
||||
|
||||
private void indexDataForScorring() {
|
||||
clearIndex();
|
||||
// 0
|
||||
assertU(add(doc("t_description", "A random movie",
|
||||
"name", "Movie 1",
|
||||
idField, "1")));
|
||||
// 1
|
||||
|
||||
assertU(add(doc("title", "The first subtitle of this movie",
|
||||
idField, "2",
|
||||
toField, "1")));
|
||||
|
||||
|
||||
// 2
|
||||
|
||||
assertU(add(doc("title", "random subtitle; random event movie",
|
||||
idField, "3",
|
||||
toField, "1")));
|
||||
|
||||
// 3
|
||||
|
||||
assertU(add(doc("t_description", "A second random movie",
|
||||
"name", "Movie 2",
|
||||
idField, "4")));
|
||||
// 4
|
||||
|
||||
assertU(add(doc("title", "a very random event happened during christmas night",
|
||||
idField, "5",
|
||||
toField, "4")));
|
||||
|
||||
|
||||
// 5
|
||||
|
||||
assertU(add(doc("title", "movie end movie test 123 test 123 random",
|
||||
idField, "6",
|
||||
toField, "4")));
|
||||
|
||||
|
||||
assertU(commit());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue