Query DSL: custom_filters_score, closes #1140.

This commit is contained in:
kimchy 2011-07-21 09:04:36 +03:00
parent de2944ff4a
commit ed6a6e31d3
8 changed files with 561 additions and 19 deletions

View File

@ -0,0 +1,257 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene.search.function;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.ComplexExplanation;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.ToStringUtils;
import org.elasticsearch.common.lucene.docset.DocSet;
import org.elasticsearch.common.lucene.docset.DocSets;
import java.io.IOException;
import java.util.Arrays;
import java.util.Set;
/**
* A query that allows for a pluggable boost function / filter. If it matches the filter, it will
* be boosted by the formula.
*
* @author kimchy (shay.banon)
*/
public class FiltersFunctionScoreQuery extends Query {
public static class FilterFunction {
public final Filter filter;
public final ScoreFunction function;
public FilterFunction(Filter filter, ScoreFunction function) {
this.filter = filter;
this.function = function;
}
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FilterFunction that = (FilterFunction) o;
if (filter != null ? !filter.equals(that.filter) : that.filter != null) return false;
if (function != null ? !function.equals(that.function) : that.function != null) return false;
return true;
}
@Override public int hashCode() {
int result = filter != null ? filter.hashCode() : 0;
result = 31 * result + (function != null ? function.hashCode() : 0);
return result;
}
}
Query subQuery;
final FilterFunction[] filterFunctions;
DocSet[] docSets;
public FiltersFunctionScoreQuery(Query subQuery, FilterFunction[] filterFunctions) {
this.subQuery = subQuery;
this.filterFunctions = filterFunctions;
this.docSets = new DocSet[filterFunctions.length];
}
public Query getSubQuery() {
return subQuery;
}
public FilterFunction[] getFilterFunctions() {
return filterFunctions;
}
@Override
public Query rewrite(IndexReader reader) throws IOException {
Query newQ = subQuery.rewrite(reader);
if (newQ == subQuery) return this;
FiltersFunctionScoreQuery bq = (FiltersFunctionScoreQuery) this.clone();
bq.subQuery = newQ;
return bq;
}
@Override
public void extractTerms(Set<Term> terms) {
subQuery.extractTerms(terms);
}
@Override
public Weight createWeight(Searcher searcher) throws IOException {
return new CustomBoostFactorWeight(searcher);
}
class CustomBoostFactorWeight extends Weight {
Searcher searcher;
Weight subQueryWeight;
public CustomBoostFactorWeight(Searcher searcher) throws IOException {
this.searcher = searcher;
this.subQueryWeight = subQuery.weight(searcher);
}
public Query getQuery() {
return FiltersFunctionScoreQuery.this;
}
public float getValue() {
return getBoost();
}
@Override
public float sumOfSquaredWeights() throws IOException {
float sum = subQueryWeight.sumOfSquaredWeights();
sum *= getBoost() * getBoost();
return sum;
}
@Override
public void normalize(float norm) {
norm *= getBoost();
subQueryWeight.normalize(norm);
}
@Override
public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
Scorer subQueryScorer = subQueryWeight.scorer(reader, scoreDocsInOrder, false);
if (subQueryScorer == null) {
return null;
}
for (int i = 0; i < filterFunctions.length; i++) {
FilterFunction filterFunction = filterFunctions[i];
filterFunction.function.setNextReader(reader);
docSets[i] = DocSets.convert(reader, filterFunction.filter.getDocIdSet(reader));
}
return new CustomBoostFactorScorer(getSimilarity(searcher), this, subQueryScorer, filterFunctions, docSets);
}
@Override
public Explanation explain(IndexReader reader, int doc) throws IOException {
Explanation subQueryExpl = subQueryWeight.explain(reader, doc);
if (!subQueryExpl.isMatch()) {
return subQueryExpl;
}
for (FilterFunction filterFunction : filterFunctions) {
DocSet docSet = DocSets.convert(reader, filterFunction.filter.getDocIdSet(reader));
if (docSet.get(doc)) {
filterFunction.function.setNextReader(reader);
Explanation functionExplanation = filterFunction.function.explain(doc, subQueryExpl);
float sc = getValue() * functionExplanation.getValue();
Explanation res = new ComplexExplanation(true, sc, "custom score, product of:");
res.addDetail(new Explanation(1.0f, "match filter: " + filterFunction.filter.toString()));
res.addDetail(functionExplanation);
res.addDetail(new Explanation(getValue(), "queryBoost"));
return res;
}
}
float sc = getValue() * subQueryExpl.getValue();
Explanation res = new ComplexExplanation(true, sc, "custom score, no filter match, product of:");
res.addDetail(subQueryExpl);
res.addDetail(new Explanation(getValue(), "queryBoost"));
return res;
}
}
static class CustomBoostFactorScorer extends Scorer {
private final float subQueryWeight;
private final Scorer scorer;
private final FilterFunction[] filterFunctions;
private final DocSet[] docSets;
private CustomBoostFactorScorer(Similarity similarity, CustomBoostFactorWeight w, Scorer scorer,
FilterFunction[] filterFunctions, DocSet[] docSets) throws IOException {
super(similarity);
this.subQueryWeight = w.getValue();
this.scorer = scorer;
this.filterFunctions = filterFunctions;
this.docSets = docSets;
}
@Override
public int docID() {
return scorer.docID();
}
@Override
public int advance(int target) throws IOException {
return scorer.advance(target);
}
@Override
public int nextDoc() throws IOException {
return scorer.nextDoc();
}
@Override
public float score() throws IOException {
int docId = scorer.docID();
float score = scorer.score();
for (int i = 0; i < filterFunctions.length; i++) {
if (docSets[i].get(docId)) {
return subQueryWeight * filterFunctions[i].function.score(docId, score);
}
}
return subQueryWeight * score;
}
}
public String toString(String field) {
StringBuilder sb = new StringBuilder();
sb.append("custom score (").append(subQuery.toString(field)).append(", functions: [");
for (FilterFunction filterFunction : filterFunctions) {
sb.append("{filter(").append(filterFunction.filter).append("), function [").append(filterFunction.function).append("]}");
}
sb.append("])");
sb.append(ToStringUtils.boost(getBoost()));
return sb.toString();
}
public boolean equals(Object o) {
if (getClass() != o.getClass()) return false;
FiltersFunctionScoreQuery other = (FiltersFunctionScoreQuery) o;
if (this.getBoost() != other.getBoost())
return false;
if (!this.subQuery.equals(other.subQuery)) {
return false;
}
return Arrays.equals(this.filterFunctions, other.filterFunctions);
}
public int hashCode() {
return subQuery.hashCode() + 31 * Arrays.hashCode(filterFunctions) ^ Float.floatToIntBits(getBoost());
}
}

View File

@ -21,7 +21,13 @@ package org.elasticsearch.common.lucene.search.function;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.search.ComplexExplanation;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
@ -106,7 +112,7 @@ public class FunctionScoreQuery extends Query {
return null;
}
function.setNextReader(reader);
return new CustomBoostFactorScorer(getSimilarity(searcher), this, subQueryScorer);
return new CustomBoostFactorScorer(getSimilarity(searcher), this, subQueryScorer, function);
}
@Override
@ -127,14 +133,16 @@ public class FunctionScoreQuery extends Query {
}
class CustomBoostFactorScorer extends Scorer {
static class CustomBoostFactorScorer extends Scorer {
private final float subQueryWeight;
private final Scorer scorer;
private final ScoreFunction function;
private CustomBoostFactorScorer(Similarity similarity, CustomBoostFactorWeight w, Scorer scorer) throws IOException {
private CustomBoostFactorScorer(Similarity similarity, CustomBoostFactorWeight w, Scorer scorer, ScoreFunction function) throws IOException {
super(similarity);
this.subQueryWeight = w.getValue();
this.scorer = scorer;
this.function = function;
}
@Override

View File

@ -0,0 +1,123 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query;
import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
/**
* A query that uses a filters with a script associated with them to compute the score.
*
* @author kimchy (shay.banon)
*/
public class CustomFiltersScoreQueryBuilder extends BaseQueryBuilder {
private final QueryBuilder queryBuilder;
private String lang;
private float boost = -1;
private Map<String, Object> params = null;
private ArrayList<FilterBuilder> filters = new ArrayList<FilterBuilder>();
private ArrayList<String> scripts = new ArrayList<String>();
public CustomFiltersScoreQueryBuilder(QueryBuilder queryBuilder) {
this.queryBuilder = queryBuilder;
}
public CustomFiltersScoreQueryBuilder add(FilterBuilder filter, String script) {
this.filters.add(filter);
this.scripts.add(script);
return this;
}
/**
* Sets the language of the script.
*/
public CustomFiltersScoreQueryBuilder lang(String lang) {
this.lang = lang;
return this;
}
/**
* Additional parameters that can be provided to the script.
*/
public CustomFiltersScoreQueryBuilder params(Map<String, Object> params) {
if (this.params == null) {
this.params = params;
} else {
this.params.putAll(params);
}
return this;
}
/**
* Additional parameters that can be provided to the script.
*/
public CustomFiltersScoreQueryBuilder param(String key, Object value) {
if (params == null) {
params = Maps.newHashMap();
}
params.put(key, value);
return this;
}
/**
* Sets the boost for this query. Documents matching this query will (in addition to the normal
* weightings) have their score multiplied by the boost provided.
*/
public CustomFiltersScoreQueryBuilder boost(float boost) {
this.boost = boost;
return this;
}
@Override protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(CustomFiltersScoreQueryParser.NAME);
builder.field("query");
queryBuilder.toXContent(builder, params);
builder.startArray("filters");
for (int i = 0; i < filters.size(); i++) {
builder.startObject();
builder.field("filter");
filters.get(i).toXContent(builder, params);
builder.field("script", scripts.get(i));
builder.endObject();
}
builder.endArray();
if (lang != null) {
builder.field("lang", lang);
}
if (this.params != null) {
builder.field("params", this.params);
}
if (boost != -1) {
builder.field("boost", boost);
}
builder.endObject();
}
}

View File

@ -0,0 +1,128 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
/**
* @author kimchy (shay.banon)
*/
public class CustomFiltersScoreQueryParser implements QueryParser {
public static final String NAME = "custom_filters_score";
@Inject public CustomFiltersScoreQueryParser() {
}
@Override public String[] names() {
return new String[]{NAME, Strings.toCamelCase(NAME)};
}
@Override public Query parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
Query query = null;
float boost = 1.0f;
String scriptLang = null;
Map<String, Object> vars = null;
ArrayList<Filter> filters = new ArrayList<Filter>();
ArrayList<String> scripts = new ArrayList<String>();
String currentFieldName = null;
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("query".equals(currentFieldName)) {
query = parseContext.parseInnerQuery();
} else if ("params".equals(currentFieldName)) {
vars = parser.map();
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("filters".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String script = null;
Filter filter = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("filter".equals(currentFieldName)) {
filter = parseContext.parseInnerFilter();
}
} else if (token.isValue()) {
if ("script".equals(currentFieldName)) {
script = parser.text();
}
}
}
if (script == null) {
throw new QueryParsingException(parseContext.index(), "[custom_filters_score] missing 'script' in filters array element");
}
if (filter == null) {
throw new QueryParsingException(parseContext.index(), "[custom_filters_score] missing 'filter' in filters array element");
}
filters.add(filter);
scripts.add(script);
}
}
} else if (token.isValue()) {
if ("lang".equals(currentFieldName)) {
scriptLang = parser.text();
} else if ("boost".equals(currentFieldName)) {
boost = parser.floatValue();
}
}
}
if (query == null) {
throw new QueryParsingException(parseContext.index(), "[custom_filters_score] requires 'query' field");
}
if (filters.isEmpty()) {
throw new QueryParsingException(parseContext.index(), "[custom_filters_score] requires 'filters' field");
}
SearchContext context = SearchContext.current();
if (context == null) {
throw new ElasticSearchIllegalStateException("No search context on going...");
}
FiltersFunctionScoreQuery.FilterFunction[] filterFunctions = new FiltersFunctionScoreQuery.FilterFunction[filters.size()];
for (int i = 0; i < filterFunctions.length; i++) {
SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, scripts.get(i), vars);
filterFunctions[i] = new FiltersFunctionScoreQuery.FilterFunction(filters.get(i), new CustomScoreQueryParser.ScriptScoreFunction(searchScript));
}
FiltersFunctionScoreQuery functionScoreQuery = new FiltersFunctionScoreQuery(query, filterFunctions);
functionScoreQuery.setBoost(boost);
return functionScoreQuery;
}
}

View File

@ -99,7 +99,7 @@ public class CustomScoreQueryParser implements QueryParser {
private final SearchScript script;
private ScriptScoreFunction(SearchScript script) {
public ScriptScoreFunction(SearchScript script) {
this.script = script;
}

View File

@ -398,6 +398,10 @@ public abstract class QueryBuilders {
return new CustomScoreQueryBuilder(queryBuilder);
}
public static CustomFiltersScoreQueryBuilder customFiltersScoreQuery(QueryBuilder queryBuilder) {
return new CustomFiltersScoreQueryBuilder(queryBuilder);
}
/**
* A more like this query that finds documents that are "like" the provided {@link MoreLikeThisQueryBuilder#likeText(String)}
* which is checked against the fields the query is constructed with.

View File

@ -58,6 +58,7 @@ public class IndicesQueriesRegistry {
addQueryParser(queryParsers, new ConstantScoreQueryParser());
addQueryParser(queryParsers, new CustomBoostFactorQueryParser());
addQueryParser(queryParsers, new CustomScoreQueryParser());
addQueryParser(queryParsers, new CustomFiltersScoreQueryParser());
addQueryParser(queryParsers, new SpanTermQueryParser());
addQueryParser(queryParsers, new SpanNotQueryParser());
addQueryParser(queryParsers, new SpanFirstQueryParser());

View File

@ -27,8 +27,11 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.Arrays;
import static org.elasticsearch.client.Requests.*;
import static org.elasticsearch.common.xcontent.XContentFactory.*;
import static org.elasticsearch.index.query.FilterBuilders.*;
import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.elasticsearch.search.builder.SearchSourceBuilder.*;
import static org.hamcrest.MatcherAssert.*;
@ -57,20 +60,7 @@ public class CustomScoreSearchTests extends AbstractNodesTests {
}
@Test public void testCustomScriptBoost() throws Exception {
// execute a search before we create an index
try {
client.prepareSearch().setQuery(termQuery("test", "value")).execute().actionGet();
assert false : "should fail";
} catch (Exception e) {
// ignore, no indices
}
try {
client.prepareSearch("test").setQuery(termQuery("test", "value")).execute().actionGet();
assert false : "should fail";
} catch (Exception e) {
// ignore, no indices
}
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().create(createIndexRequest("test")).actionGet();
client.index(indexRequest("test").type("type1").id("1")
@ -154,4 +144,35 @@ public class CustomScoreSearchTests extends AbstractNodesTests {
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
}
@Test public void testCustomFiltersScore() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
client.prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
client.prepareIndex("test", "type", "3").setSource("field", "value3").execute().actionGet();
client.prepareIndex("test", "type", "4").setSource("field", "value4").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse = client.prepareSearch("test")
.setQuery(customFiltersScoreQuery(matchAllQuery())
.add(termFilter("field", "value4"), "_score * 2")
.add(termFilter("field", "value2"), "_score * 3"))
.setExplain(true)
.execute().actionGet();
assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
assertThat(searchResponse.hits().totalHits(), equalTo(4l));
assertThat(searchResponse.hits().getAt(0).id(), equalTo("2"));
assertThat(searchResponse.hits().getAt(0).score(), equalTo(3.0f));
logger.info("--> Hit[0] {} Explanation {}", searchResponse.hits().getAt(0).id(), searchResponse.hits().getAt(0).explanation());
assertThat(searchResponse.hits().getAt(1).id(), equalTo("4"));
assertThat(searchResponse.hits().getAt(1).score(), equalTo(2.0f));
assertThat(searchResponse.hits().getAt(2).id(), anyOf(equalTo("1"), equalTo("3")));
assertThat(searchResponse.hits().getAt(2).score(), equalTo(1.0f));
assertThat(searchResponse.hits().getAt(3).id(), anyOf(equalTo("1"), equalTo("3")));
assertThat(searchResponse.hits().getAt(3).score(), equalTo(1.0f));
}
}