Merge pull request #17711 from jimferenczi/query_string_wildcard

Apply the default operator on analyzed wildcard in query_string builder
This commit is contained in:
Jim Ferenczi 2016-04-13 15:43:51 +02:00
commit 3946d3dc08
4 changed files with 146 additions and 38 deletions

View File

@ -14,7 +14,6 @@
files start to pass. -->
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queries[/\\]BlendedTermQuery.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queries[/\\]ExtendedCommonTermsQuery.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queryparser[/\\]classic[/\\]MapperQueryParser.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]search[/\\]postingshighlight[/\\]CustomPostingsHighlighter.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]search[/\\]vectorhighlight[/\\]CustomFieldQuery.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]Version.java" checks="LineLength" />
@ -1121,7 +1120,6 @@
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]HasParentQueryBuilderTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MoreLikeThisQueryBuilderTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MultiMatchQueryBuilderTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryStringQueryBuilderTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]RandomQueryBuilder.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]RangeQueryBuilderTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]ScoreModeTests.java" checks="LineLength" />

View File

@ -22,6 +22,7 @@ package org.apache.lucene.queryparser.classic;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@ -32,6 +33,7 @@ import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.automaton.RegExp;
import org.elasticsearch.common.lucene.search.Queries;
@ -105,7 +107,8 @@ public class MapperQueryParser extends QueryParser {
}
/**
* We override this one so we can get the fuzzy part to be treated as string, so people can do: "age:10~5" or "timestamp:2012-10-10~5d"
* We override this one so we can get the fuzzy part to be treated as string,
* so people can do: "age:10~5" or "timestamp:2012-10-10~5d"
*/
@Override
Query handleBareFuzzy(String qfield, Token fuzzySlop, String termImage) throws ParseException {
@ -277,7 +280,8 @@ public class MapperQueryParser extends QueryParser {
}
@Override
protected Query getRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) throws ParseException {
protected Query getRangeQuery(String field, String part1, String part2,
boolean startInclusive, boolean endInclusive) throws ParseException {
if ("*".equals(part1)) {
part1 = null;
}
@ -324,7 +328,8 @@ public class MapperQueryParser extends QueryParser {
}
}
private Query getRangeQuerySingle(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
private Query getRangeQuerySingle(String field, String part1, String part2,
boolean startInclusive, boolean endInclusive) {
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
if (lowercaseExpandedTerms && currentFieldType.tokenized()) {
@ -335,8 +340,10 @@ public class MapperQueryParser extends QueryParser {
try {
Query rangeQuery;
if (currentFieldType instanceof DateFieldMapper.DateFieldType && settings.timeZone() != null) {
DateFieldMapper.DateFieldType dateFieldType = (DateFieldMapper.DateFieldType) this.currentFieldType;
rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive, settings.timeZone(), null);
DateFieldMapper.DateFieldType dateFieldType =
(DateFieldMapper.DateFieldType) this.currentFieldType;
rangeQuery = dateFieldType.rangeQuery(part1, part2, startInclusive, endInclusive,
settings.timeZone(), null);
} else {
rangeQuery = currentFieldType.rangeQuery(part1, part2, startInclusive, endInclusive);
}
@ -393,7 +400,8 @@ public class MapperQueryParser extends QueryParser {
currentFieldType = context.fieldMapper(field);
if (currentFieldType != null) {
try {
return currentFieldType.fuzzyQuery(termStr, Fuzziness.build(minSimilarity), fuzzyPrefixLength, settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
return currentFieldType.fuzzyQuery(termStr, Fuzziness.build(minSimilarity),
fuzzyPrefixLength, settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
} catch (RuntimeException e) {
if (settings.lenient()) {
return null;
@ -408,7 +416,8 @@ public class MapperQueryParser extends QueryParser {
protected Query newFuzzyQuery(Term term, float minimumSimilarity, int prefixLength) {
String text = term.text();
int numEdits = FuzzyQuery.floatToEdits(minimumSimilarity, text.codePointCount(0, text.length()));
FuzzyQuery query = new FuzzyQuery(term, numEdits, prefixLength, settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
FuzzyQuery query = new FuzzyQuery(term, numEdits, prefixLength,
settings.fuzzyMaxExpansions(), FuzzyQuery.defaultTranspositions);
QueryParsers.setRewriteMethod(query, settings.fuzzyRewriteMethod());
return query;
}
@ -487,7 +496,7 @@ public class MapperQueryParser extends QueryParser {
if (!settings.analyzeWildcard()) {
return super.getPrefixQuery(field, termStr);
}
List<String> tlist;
List<List<String> > tlist;
// get Analyzer from superclass and tokenize the term
TokenStream source = null;
try {
@ -498,7 +507,9 @@ public class MapperQueryParser extends QueryParser {
return super.getPrefixQuery(field, termStr);
}
tlist = new ArrayList<>();
List<String> currentPos = new ArrayList<>();
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posAtt = source.addAttribute(PositionIncrementAttribute.class);
while (true) {
try {
@ -506,7 +517,14 @@ public class MapperQueryParser extends QueryParser {
} catch (IOException e) {
break;
}
tlist.add(termAtt.toString());
if (currentPos.isEmpty() == false && posAtt.getPositionIncrement() > 0) {
tlist.add(currentPos);
currentPos = new ArrayList<>();
}
currentPos.add(termAtt.toString());
}
if (currentPos.isEmpty() == false) {
tlist.add(currentPos);
}
} finally {
if (source != null) {
@ -514,15 +532,41 @@ public class MapperQueryParser extends QueryParser {
}
}
if (tlist.size() == 1) {
return super.getPrefixQuery(field, tlist.get(0));
if (tlist.size() == 1 && tlist.get(0).size() == 1) {
return super.getPrefixQuery(field, tlist.get(0).get(0));
} else {
// build a boolean query with prefix on each one...
// build a boolean query with prefix on the last position only.
List<BooleanClause> clauses = new ArrayList<>();
for (String token : tlist) {
clauses.add(new BooleanClause(super.getPrefixQuery(field, token), BooleanClause.Occur.SHOULD));
for (int pos = 0; pos < tlist.size(); pos++) {
List<String> plist = tlist.get(pos);
boolean isLastPos = (pos == tlist.size()-1);
Query posQuery;
if (plist.size() == 1) {
if (isLastPos) {
posQuery = getPrefixQuery(field, plist.get(0));
} else {
posQuery = newTermQuery(new Term(field, plist.get(0)));
}
} else if (isLastPos == false) {
// build a synonym query for terms in the same position.
Term[] terms = new Term[plist.size()];
for (int i = 0; i < plist.size(); i++) {
terms[i] = new Term(field, plist.get(i));
}
posQuery = new SynonymQuery(terms);
} else {
List<BooleanClause> innerClauses = new ArrayList<>();
for (String token : plist) {
innerClauses.add(new BooleanClause(getPrefixQuery(field, token),
BooleanClause.Occur.SHOULD));
}
posQuery = getBooleanQueryCoordDisabled(innerClauses);
}
clauses.add(new BooleanClause(posQuery,
getDefaultOperator() == Operator.AND ? BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD));
}
return getBooleanQueryCoordDisabled(clauses);
return getBooleanQuery(clauses);
}
}
@ -724,7 +768,8 @@ public class MapperQueryParser extends QueryParser {
}
Query query = null;
if (currentFieldType.tokenized() == false) {
query = currentFieldType.regexpQuery(termStr, RegExp.ALL, maxDeterminizedStates, multiTermRewriteMethod, context);
query = currentFieldType.regexpQuery(termStr, RegExp.ALL,
maxDeterminizedStates, multiTermRewriteMethod, context);
}
if (query == null) {
query = super.getRegexpQuery(field, termStr);
@ -741,7 +786,7 @@ public class MapperQueryParser extends QueryParser {
setAnalyzer(oldAnalyzer);
}
}
/**
* @deprecated review all use of this, don't rely on coord
*/

View File

@ -24,6 +24,7 @@ import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.io.JsonStringEncoder;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.spans.SpanBoostQuery;
@ -637,6 +638,13 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
assertThat(termQuery.getTerm().text().toLowerCase(Locale.ROOT), equalTo(value.toLowerCase(Locale.ROOT)));
}
protected static void assertPrefixQuery(Query query, String field, String value) {
assertThat(query, instanceOf(PrefixQuery.class));
PrefixQuery prefixQuery = (PrefixQuery) query;
assertThat(prefixQuery.getPrefix().field(), equalTo(field));
assertThat(prefixQuery.getPrefix().text(), equalTo(value));
}
/**
* Test serialization and deserialization of the test query.
*/

View File

@ -58,7 +58,8 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
}
QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(query);
if (randomBoolean()) {
queryStringQueryBuilder.defaultField(randomBoolean() ? STRING_FIELD_NAME : randomAsciiOfLengthBetween(1, 10));
queryStringQueryBuilder.defaultField(randomBoolean() ?
STRING_FIELD_NAME : randomAsciiOfLengthBetween(1, 10));
}
if (randomBoolean()) {
int numFields = randomIntBetween(1, 5);
@ -145,7 +146,8 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
}
@Override
protected void doAssertLuceneQuery(QueryStringQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
protected void doAssertLuceneQuery(QueryStringQueryBuilder queryBuilder,
Query query, QueryShardContext context) throws IOException {
if ("".equals(queryBuilder.queryString())) {
assertThat(query, instanceOf(MatchNoDocsQuery.class));
} else {
@ -173,7 +175,10 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
public void testToQueryPhraseQuery() throws IOException {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = queryStringQuery("\"term1 term2\"").defaultField(STRING_FIELD_NAME).phraseSlop(3).toQuery(createShardContext());
Query query = queryStringQuery("\"term1 term2\"")
.defaultField(STRING_FIELD_NAME)
.phraseSlop(3)
.toQuery(createShardContext());
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;
assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(1));
@ -204,7 +209,8 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
boostQuery = (BoostQuery) boostQuery.getQuery();
assertThat(boostQuery.getBoost(), equalTo(2.0f));
queryStringQuery = queryStringQuery("((" + STRING_FIELD_NAME + ":boosted^2) AND (" + STRING_FIELD_NAME + ":foo^1.5))^3");
queryStringQuery =
queryStringQuery("((" + STRING_FIELD_NAME + ":boosted^2) AND (" + STRING_FIELD_NAME + ":foo^1.5))^3");
query = queryStringQuery.toQuery(shardContext);
assertThat(query, instanceOf(BoostQuery.class));
boostQuery = (BoostQuery) query;
@ -226,27 +232,38 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
public void testToQueryMultipleTermsBooleanQuery() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = queryStringQuery("test1 test2").field(STRING_FIELD_NAME).useDisMax(false).toQuery(createShardContext());
Query query = queryStringQuery("test1 test2").field(STRING_FIELD_NAME)
.useDisMax(false)
.toQuery(createShardContext());
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bQuery = (BooleanQuery) query;
assertThat(bQuery.clauses().size(), equalTo(2));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test1")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test2")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(),
equalTo(new Term(STRING_FIELD_NAME, "test1")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(),
equalTo(new Term(STRING_FIELD_NAME, "test2")));
}
public void testToQueryMultipleFieldsBooleanQuery() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = queryStringQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(false).toQuery(createShardContext());
Query query = queryStringQuery("test").field(STRING_FIELD_NAME)
.field(STRING_FIELD_NAME_2)
.useDisMax(false)
.toQuery(createShardContext());
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bQuery = (BooleanQuery) query;
assertThat(bQuery.clauses().size(), equalTo(2));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(),
equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(),
equalTo(new Term(STRING_FIELD_NAME_2, "test")));
}
public void testToQueryMultipleFieldsDisMaxQuery() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = queryStringQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2).useDisMax(true).toQuery(createShardContext());
Query query = queryStringQuery("test").field(STRING_FIELD_NAME).field(STRING_FIELD_NAME_2)
.useDisMax(true)
.toQuery(createShardContext());
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query;
List<Query> disjuncts = disMaxQuery.getDisjuncts();
@ -260,13 +277,18 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bQuery = (BooleanQuery) query;
assertThat(bQuery.clauses().size(), equalTo(2));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 0).getTerm(),
equalTo(new Term(STRING_FIELD_NAME, "test")));
assertThat(assertBooleanSubQuery(query, TermQuery.class, 1).getTerm(),
equalTo(new Term(STRING_FIELD_NAME_2, "test")));
}
public void testToQueryDisMaxQuery() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = queryStringQuery("test").field(STRING_FIELD_NAME, 2.2f).field(STRING_FIELD_NAME_2).useDisMax(true).toQuery(createShardContext());
Query query = queryStringQuery("test").field(STRING_FIELD_NAME, 2.2f)
.field(STRING_FIELD_NAME_2)
.useDisMax(true)
.toQuery(createShardContext());
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query;
List<Query> disjuncts = disMaxQuery.getDisjuncts();
@ -274,9 +296,40 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
assertTermOrBoostQuery(disjuncts.get(1), STRING_FIELD_NAME_2, "test", 1.0f);
}
public void testToQueryPrefixQuery() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
for (Operator op : Operator.values()) {
Query query = queryStringQuery("foo-bar-foobar*")
.defaultField(STRING_FIELD_NAME)
.analyzeWildcard(true)
.analyzer("standard")
.defaultOperator(op)
.toQuery(createShardContext());
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery bq = (BooleanQuery) query;
assertThat(bq.clauses().size(), equalTo(3));
String[] expectedTerms = new String[]{"foo", "bar", "foobar"};
for (int i = 0; i < bq.clauses().size(); i++) {
BooleanClause clause = bq.clauses().get(i);
if (i != bq.clauses().size() - 1) {
assertTermQuery(clause.getQuery(), STRING_FIELD_NAME, expectedTerms[i]);
} else {
assertPrefixQuery(clause.getQuery(), STRING_FIELD_NAME, expectedTerms[i]);
}
if (op == Operator.AND) {
assertThat(clause.getOccur(), equalTo(BooleanClause.Occur.MUST));
} else {
assertThat(clause.getOccur(), equalTo(BooleanClause.Occur.SHOULD));
}
}
}
}
public void testToQueryRegExpQuery() throws Exception {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
Query query = queryStringQuery("/foo*bar/").defaultField(STRING_FIELD_NAME).maxDeterminizedStates(5000).toQuery(createShardContext());
Query query = queryStringQuery("/foo*bar/").defaultField(STRING_FIELD_NAME)
.maxDeterminizedStates(5000)
.toQuery(createShardContext());
assertThat(query, instanceOf(RegexpQuery.class));
RegexpQuery regexpQuery = (RegexpQuery) query;
assertTrue(regexpQuery.toString().contains("/foo*bar/"));
@ -344,7 +397,8 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
float mainBoost = 2.0f / randomIntBetween(3, 20);
boosts[boosts.length - 1] = mainBoost;
QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(queryString).field(STRING_FIELD_NAME)
QueryStringQueryBuilder queryStringQueryBuilder =
new QueryStringQueryBuilder(queryString).field(STRING_FIELD_NAME)
.minimumShouldMatch("2").boost(mainBoost);
Query query = queryStringQueryBuilder.toQuery(createShardContext());
@ -359,14 +413,17 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
BooleanQuery booleanQuery = (BooleanQuery) query;
assertThat(booleanQuery.getMinimumNumberShouldMatch(), equalTo(2));
assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
assertThat(booleanQuery.clauses().get(0).getQuery(), equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "foo"))));
assertThat(booleanQuery.clauses().get(0).getQuery(),
equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "foo"))));
assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
assertThat(booleanQuery.clauses().get(1).getQuery(), equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "bar"))));
assertThat(booleanQuery.clauses().get(1).getQuery(),
equalTo(new TermQuery(new Term(STRING_FIELD_NAME, "bar"))));
}
public void testToQueryPhraseQueryBoostAndSlop() throws IOException {
assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder("\"test phrase\"~2").field(STRING_FIELD_NAME, 5f);
QueryStringQueryBuilder queryStringQueryBuilder =
new QueryStringQueryBuilder("\"test phrase\"~2").field(STRING_FIELD_NAME, 5f);
Query query = queryStringQueryBuilder.toQuery(createShardContext());
assertThat(query, instanceOf(DisjunctionMaxQuery.class));
DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;