honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Always use DisjunctionMaxQuery to build cross fields disjunction (#25115) 

This commit modifies query_string, simple_query_string and multi_match queries to always use a DisjunctionMaxQuery when a disjunction over multiple fields is built. The tiebreaker is set to 1 in order to behave like the boolean query in terms of scoring.
The removal of the coord factor in Lucene 7 made this change mandatory to correctly handle minimum_should_match.

Closes #23966
This commit is contained in:
Jim Ferenczi 2017-06-08 11:18:17 +02:00 committed by GitHub
parent d6d416cacc
commit 21a57c1494
13 changed files with 233 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
View File

@ -296,27 +296,6 @@ public abstract class BlendedTermQuery extends Query {
return Objects.hash(classHash(), Arrays.hashCode(equalsTerms()));
}
public static BlendedTermQuery booleanBlendedQuery(Term[] terms) {
return booleanBlendedQuery(terms, null);
}
public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final float[] boosts) {
return new BlendedTermQuery(terms, boosts) {
@Override
protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) {
BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
for (int i = 0; i < terms.length; i++) {
Query query = new TermQuery(terms[i], ctx[i]);
if (boosts != null && boosts[i] != 1f) {
query = new BoostQuery(query, boosts[i]);
}
booleanQueryBuilder.add(query, BooleanClause.Occur.SHOULD);
}
return booleanQueryBuilder.build();
}
};
}
public static BlendedTermQuery commonTermsBlendedQuery(Term[] terms, final float[] boosts, final float maxTermFrequency) {
return new BlendedTermQuery(terms, boosts) {
@Override

106
core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.FuzzyQuery;
@ -155,7 +156,7 @@ public class MapperQueryParser extends QueryParser {
// if there is no match in the mappings.
return new MatchNoDocsQuery("empty fields");
}
if (settings.useDisMax()) {
float tiebreaker = settings.useDisMax() ? settings.tieBreaker() : 1.0f;
List<Query> queries = new ArrayList<>();
boolean added = false;
for (String mField : fields) {
@ -168,18 +169,7 @@ public class MapperQueryParser extends QueryParser {
if (!added) {
return null;
}
return new DisjunctionMaxQuery(queries, settings.tieBreaker());
} else {
List<BooleanClause> clauses = new ArrayList<>();
for (String mField : fields) {
Query q = getFieldQuerySingle(mField, queryText, quoted);
if (q != null) {
clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
}
}
if (clauses.isEmpty()) return null; // happens for stopwords
return getBooleanQuery(clauses);
}
return new DisjunctionMaxQuery(queries, tiebreaker);
} else {
return getFieldQuerySingle(field, queryText, quoted);
}
@ -255,7 +245,7 @@ public class MapperQueryParser extends QueryParser {
protected Query getFieldQuery(String field, String queryText, int slop) throws ParseException {
Collection<String> fields = extractMultiFields(field);
if (fields != null) {
if (settings.useDisMax()) {
float tiebreaker = settings.useDisMax() ? settings.tieBreaker() : 1.0f;
List<Query> queries = new ArrayList<>();
boolean added = false;
for (String mField : fields) {
@ -269,19 +259,7 @@ public class MapperQueryParser extends QueryParser {
if (!added) {
return null;
}
return new DisjunctionMaxQuery(queries, settings.tieBreaker());
} else {
List<BooleanClause> clauses = new ArrayList<>();
for (String mField : fields) {
Query q = super.getFieldQuery(mField, queryText, slop);
if (q != null) {
q = applySlop(q, slop);
clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
}
}
if (clauses.isEmpty()) return null; // happens for stopwords
return getBooleanQuery(clauses);
}
return new DisjunctionMaxQuery(queries, tiebreaker);
} else {
return super.getFieldQuery(field, queryText, slop);
}
@ -308,7 +286,7 @@ public class MapperQueryParser extends QueryParser {
return getRangeQuerySingle(fields.iterator().next(), part1, part2, startInclusive, endInclusive, context);
}
if (settings.useDisMax()) {
float tiebreaker = settings.useDisMax() ? settings.tieBreaker() : 1.0f;
List<Query> queries = new ArrayList<>();
boolean added = false;
for (String mField : fields) {
@ -321,18 +299,7 @@ public class MapperQueryParser extends QueryParser {
if (!added) {
return null;
}
return new DisjunctionMaxQuery(queries, settings.tieBreaker());
} else {
List<BooleanClause> clauses = new ArrayList<>();
for (String mField : fields) {
Query q = getRangeQuerySingle(mField, part1, part2, startInclusive, endInclusive, context);
if (q != null) {
clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
}
}
if (clauses.isEmpty()) return null; // happens for stopwords
return getBooleanQuery(clauses);
}
return new DisjunctionMaxQuery(queries, tiebreaker);
}
private Query getRangeQuerySingle(String field, String part1, String part2,
@ -367,7 +334,7 @@ public class MapperQueryParser extends QueryParser {
if (fields.size() == 1) {
return getFuzzyQuerySingle(fields.iterator().next(), termStr, minSimilarity);
}
if (settings.useDisMax()) {
float tiebreaker = settings.useDisMax() ? settings.tieBreaker() : 1.0f;
List<Query> queries = new ArrayList<>();
boolean added = false;
for (String mField : fields) {
@ -380,17 +347,7 @@ public class MapperQueryParser extends QueryParser {
if (!added) {
return null;
}
return new DisjunctionMaxQuery(queries, settings.tieBreaker());
} else {
List<BooleanClause> clauses = new ArrayList<>();
for (String mField : fields) {
Query q = getFuzzyQuerySingle(mField, termStr, minSimilarity);
if (q != null) {
clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
}
}
return getBooleanQuery(clauses);
}
return new DisjunctionMaxQuery(queries, tiebreaker);
} else {
return getFuzzyQuerySingle(field, termStr, minSimilarity);
}
@ -430,7 +387,7 @@ public class MapperQueryParser extends QueryParser {
if (fields.size() == 1) {
return getPrefixQuerySingle(fields.iterator().next(), termStr);
}
if (settings.useDisMax()) {
float tiebreaker = settings.useDisMax() ? settings.tieBreaker() : 1.0f;
List<Query> queries = new ArrayList<>();
boolean added = false;
for (String mField : fields) {
@ -443,18 +400,7 @@ public class MapperQueryParser extends QueryParser {
if (!added) {
return null;
}
return new DisjunctionMaxQuery(queries, settings.tieBreaker());
} else {
List<BooleanClause> clauses = new ArrayList<>();
for (String mField : fields) {
Query q = getPrefixQuerySingle(mField, termStr);
if (q != null) {
clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
}
}
if (clauses.isEmpty()) return null; // happens for stopwords
return getBooleanQuery(clauses);
}
return new DisjunctionMaxQuery(queries, tiebreaker);
} else {
return getPrefixQuerySingle(field, termStr);
}
@ -592,7 +538,7 @@ public class MapperQueryParser extends QueryParser {
if (fields.size() == 1) {
return getWildcardQuerySingle(fields.iterator().next(), termStr);
}
if (settings.useDisMax()) {
float tiebreaker = settings.useDisMax() ? settings.tieBreaker() : 1.0f;
List<Query> queries = new ArrayList<>();
boolean added = false;
for (String mField : fields) {
@ -605,18 +551,7 @@ public class MapperQueryParser extends QueryParser {
if (!added) {
return null;
}
return new DisjunctionMaxQuery(queries, settings.tieBreaker());
} else {
List<BooleanClause> clauses = new ArrayList<>();
for (String mField : fields) {
Query q = getWildcardQuerySingle(mField, termStr);
if (q != null) {
clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
}
}
if (clauses.isEmpty()) return null; // happens for stopwords
return getBooleanQuery(clauses);
}
return new DisjunctionMaxQuery(queries, tiebreaker);
} else {
return getWildcardQuerySingle(field, termStr);
}
@ -656,7 +591,7 @@ public class MapperQueryParser extends QueryParser {
if (fields.size() == 1) {
return getRegexpQuerySingle(fields.iterator().next(), termStr);
}
if (settings.useDisMax()) {
float tiebreaker = settings.useDisMax() ? settings.tieBreaker() : 1.0f;
List<Query> queries = new ArrayList<>();
boolean added = false;
for (String mField : fields) {
@ -669,18 +604,7 @@ public class MapperQueryParser extends QueryParser {
if (!added) {
return null;
}
return new DisjunctionMaxQuery(queries, settings.tieBreaker());
} else {
List<BooleanClause> clauses = new ArrayList<>();
for (String mField : fields) {
Query q = getRegexpQuerySingle(mField, termStr);
if (q != null) {
clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD));
}
}
if (clauses.isEmpty()) return null; // happens for stopwords
return getBooleanQuery(clauses);
}
return new DisjunctionMaxQuery(queries, tiebreaker);
} else {
return getRegexpQuerySingle(field, termStr);
}

40
core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
View File

@ -27,6 +27,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
@ -36,6 +37,7 @@ import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
import org.elasticsearch.index.mapper.MappedFieldType;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.Objects;
import java.util.List;
@ -79,18 +81,21 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
@Override
public Query newDefaultQuery(String text) {
BooleanQuery.Builder bq = new BooleanQuery.Builder();
List<Query> disjuncts = new ArrayList<>();
for (Map.Entry<String,Float> entry : weights.entrySet()) {
try {
Query q = createBooleanQuery(entry.getKey(), text, super.getDefaultOperator());
if (q != null) {
bq.add(wrapWithBoost(q, entry.getValue()), BooleanClause.Occur.SHOULD);
disjuncts.add(wrapWithBoost(q, entry.getValue()));
}
} catch (RuntimeException e) {
rethrowUnlessLenient(e);
}
}
return super.simplify(bq.build());
if (disjuncts.size() == 1) {
return disjuncts.get(0);
}
return new DisjunctionMaxQuery(disjuncts, 1.0f);
}
/**
@ -99,23 +104,26 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
*/
@Override
public Query newFuzzyQuery(String text, int fuzziness) {
BooleanQuery.Builder bq = new BooleanQuery.Builder();
List<Query> disjuncts = new ArrayList<>();
for (Map.Entry<String,Float> entry : weights.entrySet()) {
final String fieldName = entry.getKey();
try {
final BytesRef term = getAnalyzer().normalize(fieldName, text);
Query query = new FuzzyQuery(new Term(fieldName, term), fuzziness);
bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD);
disjuncts.add(wrapWithBoost(query, entry.getValue()));
} catch (RuntimeException e) {
rethrowUnlessLenient(e);
}
}
return super.simplify(bq.build());
if (disjuncts.size() == 1) {
return disjuncts.get(0);
}
return new DisjunctionMaxQuery(disjuncts, 1.0f);
}
@Override
public Query newPhraseQuery(String text, int slop) {
BooleanQuery.Builder bq = new BooleanQuery.Builder();
List<Query> disjuncts = new ArrayList<>();
for (Map.Entry<String,Float> entry : weights.entrySet()) {
try {
String field = entry.getKey();
@ -129,13 +137,16 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
Float boost = entry.getValue();
Query q = createPhraseQuery(field, text, slop);
if (q != null) {
bq.add(wrapWithBoost(q, boost), BooleanClause.Occur.SHOULD);
disjuncts.add(wrapWithBoost(q, boost));
}
} catch (RuntimeException e) {
rethrowUnlessLenient(e);
}
}
return super.simplify(bq.build());
if (disjuncts.size() == 1) {
return disjuncts.get(0);
}
return new DisjunctionMaxQuery(disjuncts, 1.0f);
}
/**
@ -144,25 +155,28 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
*/
@Override
public Query newPrefixQuery(String text) {
BooleanQuery.Builder bq = new BooleanQuery.Builder();
List<Query> disjuncts = new ArrayList<>();
for (Map.Entry<String,Float> entry : weights.entrySet()) {
final String fieldName = entry.getKey();
try {
if (settings.analyzeWildcard()) {
Query analyzedQuery = newPossiblyAnalyzedQuery(fieldName, text);
if (analyzedQuery != null) {
bq.add(wrapWithBoost(analyzedQuery, entry.getValue()), BooleanClause.Occur.SHOULD);
disjuncts.add(wrapWithBoost(analyzedQuery, entry.getValue()));
}
} else {
Term term = new Term(fieldName, getAnalyzer().normalize(fieldName, text));
Query query = new PrefixQuery(term);
bq.add(wrapWithBoost(query, entry.getValue()), BooleanClause.Occur.SHOULD);
disjuncts.add(wrapWithBoost(query, entry.getValue()));
}
} catch (RuntimeException e) {
return rethrowUnlessLenient(e);
}
}
return super.simplify(bq.build());
if (disjuncts.size() == 1) {
return disjuncts.get(0);
}
return new DisjunctionMaxQuery(disjuncts, 1.0f);
}
/**

31
core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
View File

@ -22,9 +22,6 @@ package org.elasticsearch.index.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.BlendedTermQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
@ -84,7 +81,7 @@ public class MultiMatchQuery extends MatchQuery {
queryBuilder = new QueryBuilder(tieBreaker);
break;
case CROSS_FIELDS:
queryBuilder = new CrossFieldsQueryBuilder(tieBreaker);
queryBuilder = new CrossFieldsQueryBuilder();
break;
default:
throw new IllegalStateException("No such type: " + type);
@ -99,15 +96,9 @@ public class MultiMatchQuery extends MatchQuery {
private QueryBuilder queryBuilder;
public class QueryBuilder {
protected final boolean groupDismax;
protected final float tieBreaker;
public QueryBuilder(float tieBreaker) {
this(tieBreaker != 1.0f, tieBreaker);
}
public QueryBuilder(boolean groupDismax, float tieBreaker) {
this.groupDismax = groupDismax;
this.tieBreaker = tieBreaker;
}
@ -134,19 +125,11 @@ public class MultiMatchQuery extends MatchQuery {
if (groupQuery.size() == 1) {
return groupQuery.get(0);
}
if (groupDismax) {
List<Query> queries = new ArrayList<>();
for (Query query : groupQuery) {
queries.add(query);
}
return new DisjunctionMaxQuery(queries, tieBreaker);
} else {
final BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
for (Query query : groupQuery) {
booleanQuery.add(query, BooleanClause.Occur.SHOULD);
}
return booleanQuery.build();
}
}
public Query blendTerm(Term term, MappedFieldType fieldType) {
@ -165,8 +148,8 @@ public class MultiMatchQuery extends MatchQuery {
final class CrossFieldsQueryBuilder extends QueryBuilder {
private FieldAndFieldType[] blendedFields;
CrossFieldsQueryBuilder(float tieBreaker) {
super(false, tieBreaker);
CrossFieldsQueryBuilder() {
super(0.0f);
}
@Override
@ -306,8 +289,6 @@ public class MultiMatchQuery extends MatchQuery {
blendedBoost = Arrays.copyOf(blendedBoost, i);
if (commonTermsCutoff != null) {
queries.add(BlendedTermQuery.commonTermsBlendedQuery(terms, blendedBoost, commonTermsCutoff));
} else if (tieBreaker == 1.0f) {
queries.add(BlendedTermQuery.booleanBlendedQuery(terms, blendedBoost));
} else {
queries.add(BlendedTermQuery.dismaxBlendedQuery(terms, blendedBoost, tieBreaker));
}
@ -318,11 +299,7 @@ public class MultiMatchQuery extends MatchQuery {
// best effort: add clauses that are not term queries so that they have an opportunity to match
// however their score contribution will be different
// TODO: can we improve this?
BooleanQuery.Builder bq = new BooleanQuery.Builder();
for (Query query : queries) {
bq.add(query, Occur.SHOULD);
}
return bq.build();
return new DisjunctionMaxQuery(queries, 1.0f);
}
}

60
core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java
View File

@ -54,58 +54,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
public class BlendedTermQueryTests extends ESTestCase {
public void testBooleanQuery() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
String[] firstNames = new String[]{
"simon", "paul"
};
String[] surNames = new String[]{
"willnauer", "simon"
};
for (int i = 0; i < surNames.length; i++) {
Document d = new Document();
d.add(new TextField("id", Integer.toString(i), Field.Store.YES));
d.add(new TextField("firstname", firstNames[i], Field.Store.NO));
d.add(new TextField("surname", surNames[i], Field.Store.NO));
w.addDocument(d);
}
int iters = scaledRandomIntBetween(25, 100);
for (int j = 0; j < iters; j++) {
Document d = new Document();
d.add(new TextField("id", Integer.toString(firstNames.length + j), Field.Store.YES));
d.add(new TextField("firstname", rarely() ? "some_other_name" :
"simon the sorcerer", Field.Store.NO)); // make sure length-norm is the tie-breaker
d.add(new TextField("surname", "bogus", Field.Store.NO));
w.addDocument(d);
}
w.commit();
DirectoryReader reader = DirectoryReader.open(w);
IndexSearcher searcher = setSimilarity(newSearcher(reader));
{
Term[] terms = new Term[]{new Term("firstname", "simon"), new Term("surname", "simon")};
BlendedTermQuery query = BlendedTermQuery.booleanBlendedQuery(terms);
TopDocs search = searcher.search(query, 3);
ScoreDoc[] scoreDocs = search.scoreDocs;
assertEquals(3, scoreDocs.length);
assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue());
}
{
BooleanQuery.Builder query = new BooleanQuery.Builder();
query.add(new TermQuery(new Term("firstname", "simon")), BooleanClause.Occur.SHOULD);
query.add(new TermQuery(new Term("surname", "simon")), BooleanClause.Occur.SHOULD);
TopDocs search = searcher.search(query.build(), 1);
ScoreDoc[] scoreDocs = search.scoreDocs;
assertEquals(Integer.toString(1), reader.document(scoreDocs[0].doc).getField("id").stringValue());
}
reader.close();
w.close();
dir.close();
}
public void testDismaxQuery() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
@ -183,12 +131,11 @@ public class BlendedTermQueryTests extends ESTestCase {
}
String term = TestUtil.randomRealisticUnicodeString(random(), 1, 10);
Term[] terms = toTerms(fields, term);
boolean useBoolean = random().nextBoolean();
float tieBreaker = random().nextFloat();
BlendedTermQuery query = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker);
BlendedTermQuery query = BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker);
QueryUtils.check(query);
terms = toTerms(fields, term);
BlendedTermQuery query2 = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker);
BlendedTermQuery query2 = BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker);
assertEquals(query, query2);
}
}
@ -217,8 +164,7 @@ public class BlendedTermQueryTests extends ESTestCase {
terms.add(new Term(TestUtil.randomRealisticUnicodeString(random(), 1, 10), TestUtil.randomRealisticUnicodeString(random(), 1, 10)));
}
BlendedTermQuery blendedTermQuery = random().nextBoolean() ? BlendedTermQuery.dismaxBlendedQuery(terms.toArray(new Term[0]), random().nextFloat()) :
BlendedTermQuery.booleanBlendedQuery(terms.toArray(new Term[0]));
BlendedTermQuery blendedTermQuery = BlendedTermQuery.dismaxBlendedQuery(terms.toArray(new Term[0]), random().nextFloat());
Set<Term> extracted = new HashSet<>();
IndexSearcher searcher = new IndexSearcher(new MultiReader());
searcher.createNormalizedWeight(blendedTermQuery, false).extractTerms(extracted);

26
core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
View File

@ -47,6 +47,7 @@ import java.util.Map;
import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertDisjunctionSubQuery;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.either;
import static org.hamcrest.CoreMatchers.equalTo;
@ -183,14 +184,15 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test2")));
}
public void testToQueryMultipleFieldsBooleanQuery() throws Exception {
public void testToQueryMultipleFieldsDisableDismax() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = multiMatchQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(false).toQuery(createShardContext());
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bQuery = (BooleanQuery) query;
assertThat(bQuery.clauses().size(), equalTo(2));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f));
assertThat(dQuery.getDisjuncts().size(), equalTo(2));
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
}
public void testToQueryMultipleFieldsDisMaxQuery() throws Exception {
@ -198,6 +200,7 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
Query query = multiMatchQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(true).toQuery(createShardContext());
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query;
assertThat(disMaxQuery.getTieBreakerMultiplier(), equalTo(0.0f));
List<Query> disjuncts = disMaxQuery.getDisjuncts();
assertThat(disjuncts.get(0), instanceOf(TermQuery.class));
assertThat(((TermQuery) disjuncts.get(0)).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
@ -208,11 +211,12 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
public void testToQueryFieldsWildcard() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = multiMatchQuery("test").field("mapped_str*").useDisMax(false).toQuery(createShardContext());
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bQuery = (BooleanQuery) query;
assertThat(bQuery.clauses().size(), equalTo(2));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f));
assertThat(dQuery.getDisjuncts().size(), equalTo(2));
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
}
public void testToQueryFieldMissing() throws Exception {

41
core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
View File

@ -61,11 +61,13 @@ import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertDisjunctionSubQuery;
import static org.hamcrest.CoreMatchers.either;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.containsString;
@ -270,12 +272,12 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
.field(STRING_FIELD_NAME_2)
.useDisMax(false)
.toQuery(createShardContext());
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bQuery = (BooleanQuery) query;
assertThat(bQuery.clauses().size(), equalTo(2));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(),
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery bQuery = (DisjunctionMaxQuery) query;
assertThat(bQuery.getDisjuncts().size(), equalTo(2));
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(),
equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(),
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(),
equalTo(new Term(STRING_FIELD_NAME_2, "test")));
}
@ -294,12 +296,12 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
public void testToQueryFieldsWildcard() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = queryStringQuery("test").field("mapped_str*").useDisMax(false).toQuery(createShardContext());
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bQuery = (BooleanQuery) query;
assertThat(bQuery.clauses().size(), equalTo(2));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(),
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
assertThat(dQuery.getDisjuncts().size(), equalTo(2));
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(),
equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(),
assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(),
equalTo(new Term(STRING_FIELD_NAME_2, "test")));
}
@ -397,6 +399,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
// simple multi-term
Query query = queryParser.parse("guinea pig");
Query expectedQuery = new BooleanQuery.Builder()
.add(new BooleanQuery.Builder()
.add(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), Occur.MUST)
@ -448,34 +451,34 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
// span query
query = queryParser.parse("\"that guinea pig smells\"");
expectedQuery = new BooleanQuery.Builder()
.add(new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
SpanNearQuery nearQuery = new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "that")))
.addClause(new SpanOrQuery(
.addClause(
new SpanOrQuery(
new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "guinea")))
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "pig"))).build(),
new SpanTermQuery(new Term(STRING_FIELD_NAME, "cavy"))))
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "smells")))
.build(), Occur.SHOULD)
.build();
expectedQuery = new DisjunctionMaxQuery(Collections.singletonList(nearQuery), 1.0f);
assertThat(query, Matchers.equalTo(expectedQuery));
// span query with slop
query = queryParser.parse("\"that guinea pig smells\"~2");
expectedQuery = new BooleanQuery.Builder()
.add(new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
nearQuery = new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "that")))
.addClause(new SpanOrQuery(
.addClause(
new SpanOrQuery(
new SpanNearQuery.Builder(STRING_FIELD_NAME, true)
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "guinea")))
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "pig"))).build(),
new SpanTermQuery(new Term(STRING_FIELD_NAME, "cavy"))))
.addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "smells")))
.setSlop(2)
.build(),
Occur.SHOULD)
.build();
expectedQuery = new DisjunctionMaxQuery(Collections.singletonList(nearQuery), 1.0f);
assertThat(query, Matchers.equalTo(expectedQuery));
}
}

40
core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
@ -47,6 +48,7 @@ import java.util.Set;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
@ -211,7 +213,7 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
// the remaining tests requires either a mapping that we register with types in base test setup
if (getCurrentTypes().length > 0) {
Query luceneQuery = queryBuilder.toQuery(shardContext);
assertThat(luceneQuery, instanceOf(BooleanQuery.class));
assertThat(luceneQuery, anyOf(instanceOf(BooleanQuery.class), instanceOf(DisjunctionMaxQuery.class)));
}
}
@ -229,7 +231,8 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
if ("".equals(queryBuilder.value())) {
assertThat(query, instanceOf(MatchNoDocsQuery.class));
} else if (queryBuilder.fields().size() > 1) {
assertThat(query, instanceOf(BooleanQuery.class));
assertThat(query, anyOf(instanceOf(BooleanQuery.class), instanceOf(DisjunctionMaxQuery.class)));
if (query instanceof BooleanQuery) {
BooleanQuery boolQuery = (BooleanQuery) query;
for (BooleanClause clause : boolQuery.clauses()) {
if (clause.getQuery() instanceof TermQuery) {
@ -243,16 +246,24 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
Map.Entry<String, Float> field = fieldsIterator.next();
assertTermOrBoostQuery(booleanClause.getQuery(), field.getKey(), queryBuilder.value(), field.getValue());
}
/**
* TODO:
* Test disabled because we cannot check min should match consistently:
* https://github.com/elastic/elasticsearch/issues/23966
*
if (queryBuilder.minimumShouldMatch() != null && !boolQuery.isCoordDisabled()) {
if (queryBuilder.minimumShouldMatch() != null) {
assertThat(boolQuery.getMinimumNumberShouldMatch(), greaterThan(0));
}
*
**/
} else if (query instanceof DisjunctionMaxQuery) {
DisjunctionMaxQuery maxQuery = (DisjunctionMaxQuery) query;
for (Query disjunct : maxQuery.getDisjuncts()) {
if (disjunct instanceof TermQuery) {
TermQuery inner = (TermQuery) disjunct;
assertThat(inner.getTerm().bytes().toString(), is(inner.getTerm().bytes().toString().toLowerCase(Locale.ROOT)));
}
}
assertThat(maxQuery.getDisjuncts().size(), equalTo(queryBuilder.fields().size()));
Iterator<Map.Entry<String, Float>> fieldsIterator = queryBuilder.fields().entrySet().iterator();
for (Query disjunct : maxQuery) {
Map.Entry<String, Float> field = fieldsIterator.next();
assertTermOrBoostQuery(disjunct, field.getKey(), queryBuilder.value(), field.getValue());
}
}
} else if (queryBuilder.fields().size() == 1) {
Map.Entry<String, Float> field = queryBuilder.fields().entrySet().iterator().next();
assertTermOrBoostQuery(query, field.getKey(), queryBuilder.value(), field.getValue());
@ -261,7 +272,8 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
if (ms.allEnabled()) {
assertTermQuery(query, MetaData.ALL, queryBuilder.value());
} else {
assertThat(query.getClass(), anyOf(equalTo(BooleanQuery.class), equalTo(MatchNoDocsQuery.class)));
assertThat(query.getClass(),
anyOf(equalTo(BooleanQuery.class), equalTo(DisjunctionMaxQuery.class), equalTo(MatchNoDocsQuery.class)));
}
} else {
fail("Encountered lucene query type we do not have a validation implementation for in our "
@ -337,7 +349,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
assertEquals(json, ".quote", parsed.quoteFieldSuffix());
}
@AwaitsFix(bugUrl = "Waiting on fix for minimumShouldMatch https://github.com/elastic/elasticsearch/issues/23966")
public void testMinimumShouldMatch() throws IOException {
QueryShardContext shardContext = createShardContext();
int numberOfTerms = randomIntBetween(1, 4);
@ -360,12 +371,13 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
// check special case: one term & one field should get simplified to a TermQuery
if (numberOfFields * numberOfTerms == 1) {
assertThat(query, instanceOf(TermQuery.class));
} else if (numberOfTerms == 1) {
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
} else {
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery boolQuery = (BooleanQuery) query;
int expectedMinimumShouldMatch = numberOfTerms * percent / 100;
if (numberOfTerms == 1
|| simpleQueryStringBuilder.defaultOperator().equals(Operator.AND)) {
if (simpleQueryStringBuilder.defaultOperator().equals(Operator.AND)) {
expectedMinimumShouldMatch = 0;
}
assertEquals(expectedMinimumShouldMatch, boolQuery.getMinimumNumberShouldMatch());

30
core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java
View File

@ -93,13 +93,14 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase {
Query parsedQuery = multiMatchQuery("banon").field("name.first", 2).field("name.last", 3).field("foobar").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).toQuery(queryShardContext);
try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
Query rewrittenQuery = searcher.searcher().rewrite(parsedQuery);
BooleanQuery.Builder expected = new BooleanQuery.Builder();
expected.add(new TermQuery(new Term("foobar", "banon")), BooleanClause.Occur.SHOULD);
Query tq1 = new BoostQuery(new TermQuery(new Term("name.first", "banon")), 2);
Query tq2 = new BoostQuery(new TermQuery(new Term("name.last", "banon")), 3);
expected.add(new DisjunctionMaxQuery(Arrays.<Query>asList(tq1, tq2), 0f), BooleanClause.Occur.SHOULD);
assertEquals(expected.build(), rewrittenQuery);
Query expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term("foobar", "banon")),
new DisjunctionMaxQuery(Arrays.asList(tq1, tq2), 0f)
), 0f);
assertEquals(expected, rewrittenQuery);
}
}
@ -110,7 +111,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase {
ft2.setName("bar");
Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") };
float[] boosts = new float[] {2, 3};
Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts);
Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f);
Query actual = MultiMatchQuery.blendTerm(
indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
@ -126,7 +127,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase {
ft2.setBoost(10);
Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") };
float[] boosts = new float[] {200, 30};
Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts);
Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f);
Query actual = MultiMatchQuery.blendTerm(
indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
@ -145,7 +146,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase {
ft2.setName("bar");
Term[] terms = new Term[] { new Term("foo", "baz") };
float[] boosts = new float[] {2};
Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts);
Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f);
Query actual = MultiMatchQuery.blendTerm(
indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));
@ -164,12 +165,13 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase {
ft2.setName("bar");
Term[] terms = new Term[] { new Term("foo", "baz") };
float[] boosts = new float[] {2};
Query expectedClause1 = BlendedTermQuery.booleanBlendedQuery(terms, boosts);
Query expectedClause2 = new BoostQuery(new MatchAllDocsQuery(), 3);
Query expected = new BooleanQuery.Builder()
.add(expectedClause1, Occur.SHOULD)
.add(expectedClause2, Occur.SHOULD)
.build();
Query expectedDisjunct1 = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f);
Query expectedDisjunct2 = new BoostQuery(new MatchAllDocsQuery(), 3);
Query expected = new DisjunctionMaxQuery(
Arrays.asList(
expectedDisjunct2,
expectedDisjunct1
), 1.0f);
Query actual = MultiMatchQuery.blendTerm(
indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }),
new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3));

1
core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
View File

@ -120,7 +120,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
assertSearchHits(searchResponse, "5", "6");
}
@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/23966")
public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
createIndex("test");
ensureGreen("test");

4
modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
View File

@ -244,8 +244,8 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
commonTermsQuery.add(new Term("field", "fox"));
addQuery(commonTermsQuery, documents);
BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"),
new Term("field", "brown"), new Term("field", "fox")});
BlendedTermQuery blendedTermQuery = BlendedTermQuery.dismaxBlendedQuery(new Term[]{new Term("field", "quick"),
new Term("field", "brown"), new Term("field", "fox")}, 1.0f);
addQuery(blendedTermQuery, documents);
SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true)

2
modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
View File

@ -276,7 +276,7 @@ public class QueryAnalyzerTests extends ESTestCase {
public void testExtractQueryMetadata_blendedTermQuery() {
Term[] termsArr = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")};
BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr);
BlendedTermQuery commonTermsQuery = BlendedTermQuery.dismaxBlendedQuery(termsArr, 1.0f);
Result result = analyze(commonTermsQuery);
assertThat(result.verified, is(true));
List<Term> terms = new ArrayList<>(result.terms);

9
test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.hamcrest;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
@ -515,6 +516,14 @@ public class ElasticsearchAssertions {
return subqueryType.cast(q.clauses().get(i).getQuery());
}
public static <T extends Query> T assertDisjunctionSubQuery(Query query, Class<T> subqueryType, int i) {
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery q = (DisjunctionMaxQuery) query;
assertThat(q.getDisjuncts().size(), greaterThan(i));
assertThat(q.getDisjuncts().get(i), instanceOf(subqueryType));
return subqueryType.cast(q.getDisjuncts().get(i));
}
/**
* Run the request from a given builder and check that it throws an exception of the right type
*/