mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-25 01:19:02 +00:00
* Search enhancement: pinned queries (#44345) Search enhancement: - new query type allows selected documents to be promoted above any "organic” search results. This is the first feature in a new module `search-business-rules` which will house licensed (non OSS) logic for rewriting queries according to business rules. The PinnedQueryBuilder class offers a new `pinned` query in the DSL that takes an array of promoted IDs and an “organic” query and ensures the documents with the promoted IDs rank higher than the organic matches. Closes #44074
This commit is contained in:
markharwood
GitHub
parent
0f51dd69cb
commit
7d5ab17bb2
37
docs/reference/query-dsl/pinned-query.asciidoc
Normal file
37
docs/reference/query-dsl/pinned-query.asciidoc
Normal file
@ -0,0 +1,37 @@
|
||||
[role="xpack"]
|
||||
[testenv="basic"]
|
||||
[[query-dsl-pinned-query]]
|
||||
=== Pinned Query
|
||||
Promotes selected documents to rank higher than those matching a given query.
|
||||
This feature is typically used to guide searchers to curated documents that are
|
||||
promoted over and above any "organic" matches for a search.
|
||||
The promoted or "pinned" documents are identified using the document IDs stored in
|
||||
the <<mapping-id-field,`_id`>> field.
|
||||
|
||||
==== Example request
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
GET /_search
|
||||
{
|
||||
"query": {
|
||||
"pinned" : {
|
||||
"ids" : ["1", "4", "100"],
|
||||
"organic" : {
|
||||
"match":{
|
||||
"description": "iphone"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
|
||||
[[pinned-query-top-level-parameters]]
|
||||
==== Top-level parameters for `pinned`
|
||||
|
||||
`ids`::
|
||||
An array of <<mapping-id-field, document IDs>> listed in the order they are to appear in results.
|
||||
`organic`::
|
||||
Any choice of query used to rank documents which will be ranked below the "pinned" document ids.
|
||||
@ -31,6 +31,8 @@ A query that allows to modify the score of a sub-query with a script.
|
||||
<<query-dsl-wrapper-query,`wrapper` query>>::
|
||||
A query that accepts other queries as json or yaml string.
|
||||
|
||||
<<query-dsl-pinned-query,`pinned` query>>::
|
||||
A query that promotes selected documents over others matching a given query.
|
||||
|
||||
include::distance-feature-query.asciidoc[]
|
||||
|
||||
@ -44,4 +46,6 @@ include::script-query.asciidoc[]
|
||||
|
||||
include::script-score-query.asciidoc[]
|
||||
|
||||
include::wrapper-query.asciidoc[]
|
||||
include::wrapper-query.asciidoc[]
|
||||
|
||||
include::pinned-query.asciidoc[]
|
||||
45
x-pack/plugin/search-business-rules/build.gradle
Normal file
45
x-pack/plugin/search-business-rules/build.gradle
Normal file
@ -0,0 +1,45 @@
|
||||
evaluationDependsOn(xpackModule('core'))
|
||||
|
||||
apply plugin: 'elasticsearch.esplugin'
|
||||
|
||||
esplugin {
|
||||
name 'search-business-rules'
|
||||
description 'A plugin for applying business rules to search result rankings'
|
||||
classname 'org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules'
|
||||
extendedPlugins = ['x-pack-core']
|
||||
}
|
||||
archivesBaseName = 'x-pack-searchbusinessrules'
|
||||
|
||||
|
||||
integTest.enabled = false
|
||||
|
||||
// Instead we create a separate task to run the
|
||||
// tests based on ESIntegTestCase
|
||||
task internalClusterTest(type: Test) {
|
||||
description = 'Java fantasy integration tests'
|
||||
mustRunAfter test
|
||||
|
||||
include '**/*IT.class'
|
||||
}
|
||||
|
||||
check.dependsOn internalClusterTest
|
||||
|
||||
dependencies {
|
||||
compileOnly project(path: xpackModule('core'), configuration: 'default')
|
||||
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
|
||||
testCompile project(":test:framework")
|
||||
if (isEclipse) {
|
||||
testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
|
||||
}
|
||||
}
|
||||
|
||||
// copied from CCR
|
||||
dependencyLicenses {
|
||||
ignoreSha 'x-pack-core'
|
||||
}
|
||||
|
||||
//testingConventions.naming {
|
||||
// IT {
|
||||
// baseClass "org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilderIT"
|
||||
// }
|
||||
//}
|
||||
@ -0,0 +1,175 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
package org.apache.lucene.search;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.util.Bits;
|
||||
|
||||
/**
|
||||
* A query that wraps another query and ensures scores do not exceed a maximum value
|
||||
*/
|
||||
public final class CappedScoreQuery extends Query {
|
||||
private final Query query;
|
||||
private final float maxScore;
|
||||
|
||||
/** Caps scores from the passed in Query to the supplied maxScore parameter */
|
||||
public CappedScoreQuery(Query query, float maxScore) {
|
||||
this.query = Objects.requireNonNull(query, "Query must not be null");
|
||||
if (maxScore > 0 == false) {
|
||||
throw new IllegalArgumentException(this.getClass().getName() + " maxScore must be >0, " + maxScore + " supplied.");
|
||||
}
|
||||
this.maxScore = maxScore;
|
||||
}
|
||||
|
||||
/** Returns the encapsulated query. */
|
||||
public Query getQuery() {
|
||||
return query;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query rewrite(IndexReader reader) throws IOException {
|
||||
Query rewritten = query.rewrite(reader);
|
||||
|
||||
if (rewritten != query) {
|
||||
return new CappedScoreQuery(rewritten, maxScore);
|
||||
}
|
||||
|
||||
if (rewritten.getClass() == CappedScoreQuery.class) {
|
||||
return rewritten;
|
||||
}
|
||||
|
||||
if (rewritten.getClass() == BoostQuery.class) {
|
||||
return new CappedScoreQuery(((BoostQuery) rewritten).getQuery(), maxScore);
|
||||
}
|
||||
|
||||
return super.rewrite(reader);
|
||||
}
|
||||
|
||||
/**
|
||||
* We return this as our {@link BulkScorer} so that if the CSQ wraps a query with its own optimized top-level scorer (e.g.
|
||||
* BooleanScorer) we can use that top-level scorer.
|
||||
*/
|
||||
protected static class CappedBulkScorer extends BulkScorer {
|
||||
final BulkScorer bulkScorer;
|
||||
final Weight weight;
|
||||
final float maxScore;
|
||||
|
||||
public CappedBulkScorer(BulkScorer bulkScorer, Weight weight, float maxScore) {
|
||||
this.bulkScorer = bulkScorer;
|
||||
this.weight = weight;
|
||||
this.maxScore = maxScore;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
|
||||
return bulkScorer.score(wrapCollector(collector), acceptDocs, min, max);
|
||||
}
|
||||
|
||||
private LeafCollector wrapCollector(LeafCollector collector) {
|
||||
return new FilterLeafCollector(collector) {
|
||||
@Override
|
||||
public void setScorer(Scorable scorer) throws IOException {
|
||||
// we must wrap again here, but using the scorer passed in as parameter:
|
||||
in.setScorer(new FilterScorable(scorer) {
|
||||
@Override
|
||||
public float score() throws IOException {
|
||||
return Math.min(maxScore, in.score());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setMinCompetitiveScore(float minScore) throws IOException {
|
||||
scorer.setMinCompetitiveScore(minScore);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public long cost() {
|
||||
return bulkScorer.cost();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
|
||||
final Weight innerWeight = searcher.createWeight(query, scoreMode, boost);
|
||||
if (scoreMode.needsScores()) {
|
||||
return new CappedScoreWeight(this, innerWeight, maxScore) {
|
||||
@Override
|
||||
public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
|
||||
final BulkScorer innerScorer = innerWeight.bulkScorer(context);
|
||||
if (innerScorer == null) {
|
||||
return null;
|
||||
}
|
||||
return new CappedBulkScorer(innerScorer, this, maxScore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
|
||||
ScorerSupplier innerScorerSupplier = innerWeight.scorerSupplier(context);
|
||||
if (innerScorerSupplier == null) {
|
||||
return null;
|
||||
}
|
||||
return new ScorerSupplier() {
|
||||
@Override
|
||||
public Scorer get(long leadCost) throws IOException {
|
||||
final Scorer innerScorer = innerScorerSupplier.get(leadCost);
|
||||
// short-circuit if scores will not need capping
|
||||
innerScorer.advanceShallow(0);
|
||||
if (innerScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS) <= maxScore) {
|
||||
return innerScorer;
|
||||
}
|
||||
return new CappedScorer(innerWeight, innerScorer, maxScore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long cost() {
|
||||
return innerScorerSupplier.cost();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public Matches matches(LeafReaderContext context, int doc) throws IOException {
|
||||
return innerWeight.matches(context, doc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Scorer scorer(LeafReaderContext context) throws IOException {
|
||||
ScorerSupplier scorerSupplier = scorerSupplier(context);
|
||||
if (scorerSupplier == null) {
|
||||
return null;
|
||||
}
|
||||
return scorerSupplier.get(Long.MAX_VALUE);
|
||||
}
|
||||
};
|
||||
} else {
|
||||
return innerWeight;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString(String field) {
|
||||
return new StringBuilder("CappedScore(").append(query.toString(field)).append(')').toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
return sameClassAs(other) && maxScore == ((CappedScoreQuery) other).maxScore &&
|
||||
query.equals(((CappedScoreQuery) other).query);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return 31 * classHash() + query.hashCode() + Float.hashCode(maxScore);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.apache.lucene.search;
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.Term;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* A Weight that caps scores of the wrapped query to a maximum value
|
||||
*/
|
||||
public abstract class CappedScoreWeight extends Weight {
|
||||
|
||||
private final float maxScore;
|
||||
private final Weight innerWeight;
|
||||
|
||||
protected CappedScoreWeight(Query query, Weight innerWeight, float maxScore) {
|
||||
super(query);
|
||||
this.maxScore = maxScore;
|
||||
this.innerWeight = innerWeight;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void extractTerms(Set<Term> terms) {
|
||||
innerWeight.extractTerms(terms);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isCacheable(LeafReaderContext ctx) {
|
||||
return innerWeight.isCacheable(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Scorer scorer(LeafReaderContext context) throws IOException {
|
||||
return new CappedScorer(this, innerWeight.scorer(context), maxScore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
|
||||
|
||||
final Scorer s = scorer(context);
|
||||
final boolean exists;
|
||||
if (s == null) {
|
||||
exists = false;
|
||||
} else {
|
||||
final TwoPhaseIterator twoPhase = s.twoPhaseIterator();
|
||||
if (twoPhase == null) {
|
||||
exists = s.iterator().advance(doc) == doc;
|
||||
} else {
|
||||
exists = twoPhase.approximation().advance(doc) == doc && twoPhase.matches();
|
||||
}
|
||||
}
|
||||
|
||||
Explanation sub = innerWeight.explain(context, doc);
|
||||
if (sub.isMatch() && sub.getValue().floatValue() > maxScore) {
|
||||
return Explanation.match(maxScore, "Capped score of " + innerWeight.getQuery() + ", max of",
|
||||
sub,
|
||||
Explanation.match(maxScore, "maximum score"));
|
||||
} else {
|
||||
return sub;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.apache.lucene.search;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class CappedScorer extends FilterScorer {
|
||||
private final float maxScore;
|
||||
|
||||
public CappedScorer(Weight weight, Scorer delegate, float maxScore) {
|
||||
super(delegate, weight);
|
||||
this.maxScore = maxScore;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getMaxScore(int upTo) throws IOException {
|
||||
return Math.min(maxScore, in.getMaxScore(upTo));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int advanceShallow(int target) throws IOException {
|
||||
return in.advanceShallow(target);
|
||||
}
|
||||
|
||||
@Override
|
||||
public float score() throws IOException {
|
||||
return Math.min(maxScore, in.score());
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,220 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.searchbusinessrules;
|
||||
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.BoostQuery;
|
||||
import org.apache.lucene.search.CappedScoreQuery;
|
||||
import org.apache.lucene.search.ConstantScoreQuery;
|
||||
import org.apache.lucene.search.DisjunctionMaxQuery;
|
||||
import org.apache.lucene.search.MatchNoDocsQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.util.NumericUtils;
|
||||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.common.ParsingException;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.mapper.IdFieldMapper;
|
||||
import org.elasticsearch.index.mapper.MappedFieldType;
|
||||
import org.elasticsearch.index.query.AbstractQueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryRewriteContext;
|
||||
import org.elasticsearch.index.query.QueryShardContext;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
|
||||
|
||||
/**
|
||||
* A query that will promote selected documents (identified by ID) above matches produced by an "organic" query. In practice, some upstream
|
||||
* system will identify the promotions associated with a user's query string and use this object to ensure these are "pinned" to the top of
|
||||
* the other search results.
|
||||
*/
|
||||
public class PinnedQueryBuilder extends AbstractQueryBuilder<PinnedQueryBuilder> {
|
||||
public static final String NAME = "pinned";
|
||||
public static final int MAX_NUM_PINNED_HITS = 100;
|
||||
|
||||
private static final ParseField IDS_FIELD = new ParseField("ids");
|
||||
public static final ParseField ORGANIC_QUERY_FIELD = new ParseField("organic");
|
||||
|
||||
private final List<String> ids;
|
||||
private QueryBuilder organicQuery;
|
||||
|
||||
// Organic queries will have their scores capped to this number range,
|
||||
// We reserve the highest float exponent for scores of pinned queries
|
||||
private static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1;
|
||||
|
||||
/**
|
||||
* Creates a new PinnedQueryBuilder
|
||||
*/
|
||||
public PinnedQueryBuilder(QueryBuilder organicQuery, String... ids) {
|
||||
if (organicQuery == null) {
|
||||
throw new IllegalArgumentException("[" + NAME + "] organicQuery cannot be null");
|
||||
}
|
||||
this.organicQuery = organicQuery;
|
||||
if (ids == null) {
|
||||
throw new IllegalArgumentException("[" + NAME + "] ids cannot be null");
|
||||
}
|
||||
if (ids.length > MAX_NUM_PINNED_HITS) {
|
||||
throw new IllegalArgumentException("[" + NAME + "] Max of "+MAX_NUM_PINNED_HITS+" ids exceeded: "+
|
||||
ids.length+" provided.");
|
||||
}
|
||||
LinkedHashSet<String> deduped = new LinkedHashSet<>();
|
||||
for (String id : ids) {
|
||||
if (id == null) {
|
||||
throw new IllegalArgumentException("[" + NAME + "] id cannot be null");
|
||||
}
|
||||
if(deduped.add(id) == false) {
|
||||
throw new IllegalArgumentException("[" + NAME + "] duplicate id found in list: "+id);
|
||||
}
|
||||
}
|
||||
this.ids = new ArrayList<>();
|
||||
Collections.addAll(this.ids, ids);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Read from a stream.
|
||||
*/
|
||||
public PinnedQueryBuilder(StreamInput in) throws IOException {
|
||||
super(in);
|
||||
ids = in.readStringList();
|
||||
organicQuery = in.readNamedWriteable(QueryBuilder.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doWriteTo(StreamOutput out) throws IOException {
|
||||
out.writeStringCollection(this.ids);
|
||||
out.writeNamedWriteable(organicQuery);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the organic query set in the constructor
|
||||
*/
|
||||
public QueryBuilder organicQuery() {
|
||||
return this.organicQuery;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the pinned ids for the query.
|
||||
*/
|
||||
public List<String> ids() {
|
||||
return Collections.unmodifiableList(this.ids);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
builder.startObject(NAME);
|
||||
if (organicQuery != null) {
|
||||
builder.field(ORGANIC_QUERY_FIELD.getPreferredName());
|
||||
organicQuery.toXContent(builder, params);
|
||||
}
|
||||
builder.startArray(IDS_FIELD.getPreferredName());
|
||||
for (String value : ids) {
|
||||
builder.value(value);
|
||||
}
|
||||
builder.endArray();
|
||||
printBoostAndQueryName(builder);
|
||||
builder.endObject();
|
||||
}
|
||||
|
||||
|
||||
|
||||
private static final ConstructingObjectParser<PinnedQueryBuilder, XContentParser> PARSER = new ConstructingObjectParser<>(NAME,
|
||||
a ->
|
||||
{
|
||||
QueryBuilder organicQuery = (QueryBuilder) a[0];
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> ids = (List<String>) a[1];
|
||||
return new PinnedQueryBuilder(organicQuery, ids.toArray(new String[0]));
|
||||
}
|
||||
);
|
||||
static {
|
||||
PARSER.declareObject(constructorArg(), (p, c) -> parseInnerQueryBuilder(p), ORGANIC_QUERY_FIELD);
|
||||
PARSER.declareStringArray(constructorArg(), IDS_FIELD);
|
||||
declareStandardFields(PARSER);
|
||||
}
|
||||
|
||||
public static PinnedQueryBuilder fromXContent(XContentParser parser) {
|
||||
try {
|
||||
return PARSER.apply(parser, null);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getWriteableName() {
|
||||
return NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
|
||||
QueryBuilder newOrganicQuery = organicQuery.rewrite(queryShardContext);
|
||||
if (newOrganicQuery != organicQuery) {
|
||||
PinnedQueryBuilder result = new PinnedQueryBuilder(newOrganicQuery, ids.toArray(new String[0]));
|
||||
result.boost(this.boost);
|
||||
return result;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Query doToQuery(QueryShardContext context) throws IOException {
|
||||
MappedFieldType idField = context.fieldMapper(IdFieldMapper.NAME);
|
||||
if (idField == null) {
|
||||
return new MatchNoDocsQuery("No mappings");
|
||||
}
|
||||
if (this.ids.isEmpty()) {
|
||||
return new CappedScoreQuery(organicQuery.toQuery(context), MAX_ORGANIC_SCORE);
|
||||
} else {
|
||||
BooleanQuery.Builder pinnedQueries = new BooleanQuery.Builder();
|
||||
|
||||
// Ensure each pin order using a Boost query with the relevant boost factor
|
||||
int minPin = NumericUtils.floatToSortableInt(MAX_ORGANIC_SCORE) + 1;
|
||||
int boostNum = minPin + ids.size();
|
||||
float lastScore = Float.MAX_VALUE;
|
||||
for (String id : ids) {
|
||||
float pinScore = NumericUtils.sortableIntToFloat(boostNum);
|
||||
assert pinScore < lastScore;
|
||||
lastScore = pinScore;
|
||||
boostNum--;
|
||||
// Ensure the pin order using a Boost query with the relevant boost factor
|
||||
Query idQuery = new BoostQuery(new ConstantScoreQuery(idField.termQuery(id, context)), pinScore);
|
||||
pinnedQueries.add(idQuery, BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
|
||||
// Score for any pinned query clause should be used, regardless of any organic clause score, to preserve pin order.
|
||||
// Use dismax to always take the larger (ie pinned) of the organic vs pinned scores
|
||||
List<Query> organicAndPinned = new ArrayList<>();
|
||||
organicAndPinned.add(pinnedQueries.build());
|
||||
// Cap the scores of the organic query
|
||||
organicAndPinned.add(new CappedScoreQuery(organicQuery.toQuery(context), MAX_ORGANIC_SCORE));
|
||||
return new DisjunctionMaxQuery(organicAndPinned, 0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int doHashCode() {
|
||||
return Objects.hash(ids, organicQuery);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean doEquals(PinnedQueryBuilder other) {
|
||||
return Objects.equals(ids, other.ids) && Objects.equals(organicQuery, other.organicQuery) && boost == other.boost;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.searchbusinessrules;
|
||||
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.plugins.SearchPlugin;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
|
||||
public class SearchBusinessRules extends Plugin implements SearchPlugin {
|
||||
|
||||
@Override
|
||||
public List<QuerySpec<?>> getQueries() {
|
||||
return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,173 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.searchbusinessrules;
|
||||
|
||||
import org.apache.lucene.search.Explanation;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.search.SearchType;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.query.MatchAllQueryBuilder;
|
||||
import org.elasticsearch.index.query.Operator;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThirdHit;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.lessThan;
|
||||
import static org.hamcrest.Matchers.lessThanOrEqualTo;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
|
||||
|
||||
public class PinnedQueryBuilderIT extends ESIntegTestCase {
|
||||
|
||||
public void testIdInsertionOrderRetained() {
|
||||
String[] ids = generateRandomStringArray(10, 50, false);
|
||||
PinnedQueryBuilder pqb = new PinnedQueryBuilder(new MatchAllQueryBuilder(), ids);
|
||||
List<String> addedIds = pqb.ids();
|
||||
int pos = 0;
|
||||
for (String key : addedIds) {
|
||||
assertEquals(ids[pos++], key);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> nodePlugins() {
|
||||
List<Class<? extends Plugin>> plugins = new ArrayList<>();
|
||||
plugins.add(SearchBusinessRules.class);
|
||||
return plugins;
|
||||
}
|
||||
|
||||
public void testPinnedPromotions() throws Exception {
|
||||
assertAcked(prepareCreate("test")
|
||||
.addMapping("type1",
|
||||
jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("field1")
|
||||
.field("analyzer", "whitespace").field("type", "text").endObject().endObject().endObject().endObject())
|
||||
.setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 2)));
|
||||
|
||||
int numRelevantDocs = randomIntBetween(1, 100);
|
||||
for (int i = 0; i < numRelevantDocs; i++) {
|
||||
if (i % 2 == 0) {
|
||||
// add lower-scoring text
|
||||
client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "the quick brown fox").get();
|
||||
} else {
|
||||
// add higher-scoring text
|
||||
client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "red fox").get();
|
||||
}
|
||||
}
|
||||
// Add docs with no relevance
|
||||
int numIrrelevantDocs = randomIntBetween(1, 10);
|
||||
for (int i = numRelevantDocs; i <= numRelevantDocs + numIrrelevantDocs; i++) {
|
||||
client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "irrelevant").get();
|
||||
}
|
||||
refresh();
|
||||
|
||||
// Test doc pinning
|
||||
int totalDocs = numRelevantDocs + numIrrelevantDocs;
|
||||
for (int i = 0; i < 100; i++) {
|
||||
int numPromotions = randomIntBetween(0, totalDocs);
|
||||
|
||||
LinkedHashSet<String> pins = new LinkedHashSet<>();
|
||||
for (int j = 0; j < numPromotions; j++) {
|
||||
pins.add(Integer.toString(randomIntBetween(0, totalDocs)));
|
||||
}
|
||||
QueryBuilder organicQuery = null;
|
||||
if (i % 5 == 0) {
|
||||
// Occasionally try a query with no matches to check all pins still show
|
||||
organicQuery = QueryBuilders.matchQuery("field1", "matchNoDocs");
|
||||
} else {
|
||||
organicQuery = QueryBuilders.matchQuery("field1", "red fox");
|
||||
}
|
||||
PinnedQueryBuilder pqb = new PinnedQueryBuilder(organicQuery, pins.toArray(new String[0]));
|
||||
|
||||
int from = randomIntBetween(0, numRelevantDocs);
|
||||
int size = randomIntBetween(10, 100);
|
||||
SearchResponse searchResponse = client().prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSize(size).setFrom(from)
|
||||
.get();
|
||||
|
||||
long numHits = searchResponse.getHits().getTotalHits().value;
|
||||
assertThat(numHits, lessThanOrEqualTo((long) numRelevantDocs + pins.size()));
|
||||
|
||||
// Check pins are sorted by increasing score, (unlike organic, there are no duplicate scores)
|
||||
float lastScore = Float.MAX_VALUE;
|
||||
SearchHit[] hits = searchResponse.getHits().getHits();
|
||||
for (int hitNumber = 0; hitNumber < Math.min(hits.length, pins.size() - from); hitNumber++) {
|
||||
assertThat("Hit " + hitNumber + " in iter " + i + " wrong" + pins, hits[hitNumber].getScore(), lessThan(lastScore));
|
||||
lastScore = hits[hitNumber].getScore();
|
||||
}
|
||||
// Check that the pins appear in the requested order (globalHitNumber is cursor independent of from and size window used)
|
||||
int globalHitNumber = 0;
|
||||
for (String id : pins) {
|
||||
if (globalHitNumber < size && globalHitNumber >= from) {
|
||||
assertThat("Hit " + globalHitNumber + " in iter " + i + " wrong" + pins, hits[globalHitNumber - from].getId(),
|
||||
equalTo(id));
|
||||
}
|
||||
globalHitNumber++;
|
||||
}
|
||||
// Test the organic hits are sorted by text relevance
|
||||
boolean highScoresExhausted = false;
|
||||
for (; globalHitNumber < hits.length + from; globalHitNumber++) {
|
||||
if (globalHitNumber >= from) {
|
||||
int id = Integer.parseInt(hits[globalHitNumber - from].getId());
|
||||
if (id % 2 == 0) {
|
||||
highScoresExhausted = true;
|
||||
} else {
|
||||
assertFalse("All odd IDs should have scored higher than even IDs in organic results", highScoresExhausted);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void testExplain() throws Exception {
|
||||
assertAcked(prepareCreate("test").addMapping("type1",
|
||||
jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("field1")
|
||||
.field("analyzer", "whitespace").field("type", "text").endObject().endObject().endObject().endObject()));
|
||||
ensureGreen();
|
||||
client().prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox").get();
|
||||
client().prepareIndex("test", "type1", "2").setSource("field1", "pinned").get();
|
||||
client().prepareIndex("test", "type1", "3").setSource("field1", "irrelevant").get();
|
||||
client().prepareIndex("test", "type1", "4").setSource("field1", "slow brown cat").get();
|
||||
refresh();
|
||||
|
||||
PinnedQueryBuilder pqb = new PinnedQueryBuilder(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR), "2");
|
||||
|
||||
SearchResponse searchResponse = client().prepareSearch().setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(pqb)
|
||||
.setExplain(true).get();
|
||||
assertHitCount(searchResponse, 3);
|
||||
assertFirstHit(searchResponse, hasId("2"));
|
||||
assertSecondHit(searchResponse, hasId("1"));
|
||||
assertThirdHit(searchResponse, hasId("4"));
|
||||
|
||||
Explanation pinnedExplanation = searchResponse.getHits().getAt(0).getExplanation();
|
||||
assertThat(pinnedExplanation, notNullValue());
|
||||
assertThat(pinnedExplanation.isMatch(), equalTo(true));
|
||||
assertThat(pinnedExplanation.getDetails().length, equalTo(1));
|
||||
assertThat(pinnedExplanation.getDetails()[0].isMatch(), equalTo(true));
|
||||
assertThat(pinnedExplanation.getDetails()[0].getDescription(), containsString("ConstantScore"));
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,173 @@
|
||||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.searchbusinessrules;
|
||||
|
||||
import com.fasterxml.jackson.core.io.JsonStringEncoder;
|
||||
|
||||
import org.apache.lucene.search.DisjunctionMaxQuery;
|
||||
import org.apache.lucene.search.MatchNoDocsQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.elasticsearch.common.ParsingException;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.index.query.MatchAllQueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.TermQueryBuilder;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
import org.elasticsearch.test.AbstractQueryTestCase;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
|
||||
public class PinnedQueryBuilderTests extends AbstractQueryTestCase<PinnedQueryBuilder> {
|
||||
@Override
|
||||
protected PinnedQueryBuilder doCreateTestQueryBuilder() {
|
||||
return new PinnedQueryBuilder(createRandomQuery(), generateRandomStringArray(100, 256, false, true));
|
||||
}
|
||||
|
||||
private QueryBuilder createRandomQuery() {
|
||||
if (randomBoolean()) {
|
||||
return new MatchAllQueryBuilder();
|
||||
} else {
|
||||
return createTestTermQueryBuilder();
|
||||
}
|
||||
}
|
||||
|
||||
private QueryBuilder createTestTermQueryBuilder() {
|
||||
String fieldName = null;
|
||||
Object value;
|
||||
switch (randomIntBetween(0, 3)) {
|
||||
case 0:
|
||||
if (randomBoolean()) {
|
||||
fieldName = BOOLEAN_FIELD_NAME;
|
||||
}
|
||||
value = randomBoolean();
|
||||
break;
|
||||
case 1:
|
||||
if (randomBoolean()) {
|
||||
fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME);
|
||||
}
|
||||
if (frequently()) {
|
||||
value = randomAlphaOfLengthBetween(1, 10);
|
||||
} else {
|
||||
// generate unicode string in 10% of cases
|
||||
JsonStringEncoder encoder = JsonStringEncoder.getInstance();
|
||||
value = new String(encoder.quoteAsString(randomUnicodeOfLength(10)));
|
||||
}
|
||||
break;
|
||||
case 2:
|
||||
if (randomBoolean()) {
|
||||
fieldName = INT_FIELD_NAME;
|
||||
}
|
||||
value = randomInt(10000);
|
||||
break;
|
||||
case 3:
|
||||
if (randomBoolean()) {
|
||||
fieldName = DOUBLE_FIELD_NAME;
|
||||
}
|
||||
value = randomDouble();
|
||||
break;
|
||||
default:
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
if (fieldName == null) {
|
||||
fieldName = randomAlphaOfLengthBetween(1, 10);
|
||||
}
|
||||
return new TermQueryBuilder(fieldName, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doAssertLuceneQuery(PinnedQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException {
|
||||
if (queryBuilder.ids().size() == 0 && queryBuilder.organicQuery() == null) {
|
||||
assertThat(query, instanceOf(MatchNoDocsQuery.class));
|
||||
} else {
|
||||
if (queryBuilder.ids().size() > 0) {
|
||||
// Have IDs and an organic query - uses DisMax
|
||||
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> getPlugins() {
|
||||
List<Class<? extends Plugin>> classpathPlugins = new ArrayList<>();
|
||||
classpathPlugins.add(SearchBusinessRules.class);
|
||||
return classpathPlugins;
|
||||
}
|
||||
|
||||
public void testIllegalArguments() {
|
||||
expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), (String)null));
|
||||
expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(null, "1"));
|
||||
expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), "1", null, "2"));
|
||||
String[] bigList = new String[PinnedQueryBuilder.MAX_NUM_PINNED_HITS + 1];
|
||||
for (int i = 0; i < bigList.length; i++) {
|
||||
bigList[i] = String.valueOf(i);
|
||||
}
|
||||
expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), bigList));
|
||||
|
||||
}
|
||||
|
||||
public void testEmptyPinnedQuery() throws Exception {
|
||||
XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
|
||||
contentBuilder.startObject().startObject("pinned").endObject().endObject();
|
||||
try (XContentParser xParser = createParser(contentBuilder)) {
|
||||
expectThrows(ParsingException.class, () -> parseQuery(xParser).toQuery(createShardContext()));
|
||||
}
|
||||
}
|
||||
|
||||
public void testFromJson() throws IOException {
|
||||
String query =
|
||||
"{" +
|
||||
"\"pinned\" : {" +
|
||||
" \"organic\" : {" +
|
||||
" \"term\" : {" +
|
||||
" \"tag\" : {" +
|
||||
" \"value\" : \"tech\"," +
|
||||
" \"boost\" : 1.0" +
|
||||
" }" +
|
||||
" }" +
|
||||
" }, "+
|
||||
" \"ids\" : [ \"1\",\"2\" ]," +
|
||||
" \"boost\":1.0 "+
|
||||
"}" +
|
||||
"}";
|
||||
|
||||
PinnedQueryBuilder queryBuilder = (PinnedQueryBuilder) parseQuery(query);
|
||||
checkGeneratedJson(query, queryBuilder);
|
||||
|
||||
assertEquals(query, 2, queryBuilder.ids().size());
|
||||
assertThat(queryBuilder.organicQuery(), instanceOf(TermQueryBuilder.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* test that unknown query names in the clauses throw an error
|
||||
*/
|
||||
public void testUnknownQueryName() throws IOException {
|
||||
String query = "{\"pinned\" : {\"organic\" : { \"unknown_query\" : { } } } }";
|
||||
|
||||
ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query));
|
||||
// BoolQueryBuilder test has this test for a more detailed error message:
|
||||
// assertEquals("no [query] registered for [unknown_query]", ex.getMessage());
|
||||
// But ObjectParser used in PinnedQueryBuilder tends to hide the above message and give this below:
|
||||
assertEquals("[1:46] [pinned] failed to parse field [organic]", ex.getMessage());
|
||||
}
|
||||
|
||||
public void testRewrite() throws IOException {
|
||||
PinnedQueryBuilder pinnedQueryBuilder = new PinnedQueryBuilder(new TermQueryBuilder("foo", 1), "1");
|
||||
QueryBuilder rewritten = pinnedQueryBuilder.rewrite(createShardContext());
|
||||
assertThat(rewritten, instanceOf(PinnedQueryBuilder.class));
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user