Refactors TermsQueryBuilder and Parser

Refactors TermsQueryBuilder and Parser for #10217.

This PR is against the query-refactoring branch.

Closes #12042
This commit is contained in:
Alex Ksikes 2015-07-06 09:22:09 +02:00
parent efadf87371
commit 1af0a39221
12 changed files with 834 additions and 264 deletions

View File

@ -43,12 +43,15 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.search.termslookup.TermsLookupFetchService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.cache.query.terms.TermsLookup;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.script.ScriptService;
import java.io.IOException;
import java.util.List;
public class IndexQueryParserService extends AbstractIndexComponent {
@ -89,6 +92,8 @@ public class IndexQueryParserService extends AbstractIndexComponent {
private final ParseFieldMatcher parseFieldMatcher;
private final boolean defaultAllowUnmappedFields;
private TermsLookupFetchService termsLookupFetchService;
@Inject
public IndexQueryParserService(Index index, @IndexSettings Settings indexSettings,
IndicesQueriesRegistry indicesQueriesRegistry,
@ -115,6 +120,11 @@ public class IndexQueryParserService extends AbstractIndexComponent {
this.indicesQueriesRegistry = indicesQueriesRegistry;
}
@Inject(optional=true)
public void setTermsLookupFetchService(@Nullable TermsLookupFetchService termsLookupFetchService) {
this.termsLookupFetchService = termsLookupFetchService;
}
public void close() {
cache.close();
}
@ -339,4 +349,8 @@ public class IndexQueryParserService extends AbstractIndexComponent {
}
return false;
}
public List<Object> handleTermsLookup(TermsLookup termsLookup) {
return this.termsLookupFetchService.fetch(termsLookup);
}
}

View File

@ -603,7 +603,7 @@ public abstract class QueryBuilders {
* A terms query that can extract the terms from another doc in an index.
*/
public static TermsQueryBuilder termsLookupQuery(String name) {
return new TermsQueryBuilder(name, (Object[]) null);
return new TermsQueryBuilder(name);
}
/**

View File

@ -327,5 +327,4 @@ public class QueryShardContext {
public boolean matchesIndices(String... indices) {
return this.indexQueryParserService().matchesIndices(indices);
}
}

View File

@ -19,9 +19,30 @@
package org.elasticsearch.index.query;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Floats;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.indices.cache.query.terms.TermsLookup;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
/**
* A filter for a field based on several terms matching on any of them.
@ -30,96 +51,105 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
public static final String NAME = "terms";
static final TermsQueryBuilder PROTOTYPE = new TermsQueryBuilder(null, (Object) null);
static final TermsQueryBuilder PROTOTYPE = new TermsQueryBuilder(null);
private final String name;
private final Object values;
public static final boolean DEFAULT_DISABLE_COORD = false;
private final String fieldName;
private List<Object> values;
private String minimumShouldMatch;
private Boolean disableCoord;
private String lookupIndex;
private String lookupType;
private String lookupId;
private String lookupRouting;
private String lookupPath;
private boolean disableCoord = DEFAULT_DISABLE_COORD;
private TermsLookup termsLookup;
/**
* A filter for a field based on several terms matching on any of them.
*
* @param name The field name
* @param fieldName The field name
* @param values The terms
*/
public TermsQueryBuilder(String name, String... values) {
this(name, (Object[]) values);
public TermsQueryBuilder(String fieldName, String... values) {
this(fieldName, values != null ? Arrays.asList(values) : (Iterable<?>) null);
}
/**
* A filter for a field based on several terms matching on any of them.
*
* @param fieldName The field name
* @param values The terms
*/
public TermsQueryBuilder(String fieldName, int... values) {
this(fieldName, values != null ? Ints.asList(values) : (Iterable<?>) null);
}
/**
* A filter for a field based on several terms matching on any of them.
*
* @param name The field name
* @param fieldName The field name
* @param values The terms
*/
public TermsQueryBuilder(String name, int... values) {
this.name = name;
this.values = values;
public TermsQueryBuilder(String fieldName, long... values) {
this(fieldName, values != null ? Longs.asList(values) : (Iterable<?>) null);
}
/**
* A filter for a field based on several terms matching on any of them.
*
* @param name The field name
* @param fieldName The field name
* @param values The terms
*/
public TermsQueryBuilder(String name, long... values) {
this.name = name;
this.values = values;
public TermsQueryBuilder(String fieldName, float... values) {
this(fieldName, values != null ? Floats.asList(values) : (Iterable<?>) null);
}
/**
* A filter for a field based on several terms matching on any of them.
*
* @param name The field name
* @param fieldName The field name
* @param values The terms
*/
public TermsQueryBuilder(String name, float... values) {
this.name = name;
this.values = values;
public TermsQueryBuilder(String fieldName, double... values) {
this(fieldName, values != null ? Doubles.asList(values) : (Iterable<?>) null);
}
/**
* A filter for a field based on several terms matching on any of them.
*
* @param name The field name
* @param fieldName The field name
* @param values The terms
*/
public TermsQueryBuilder(String name, double... values) {
this.name = name;
this.values = values;
public TermsQueryBuilder(String fieldName, Object... values) {
this(fieldName, values != null ? Arrays.asList(values) : (Iterable<?>) null);
}
/**
* Constructor used for terms query lookup.
*
* @param fieldName The field name
*/
public TermsQueryBuilder(String fieldName) {
this.fieldName = fieldName;
}
/**
* A filter for a field based on several terms matching on any of them.
*
* @param name The field name
* @param fieldName The field name
* @param values The terms
*/
public TermsQueryBuilder(String name, Object... values) {
this.name = name;
this.values = values;
public TermsQueryBuilder(String fieldName, Iterable<?> values) {
if (values == null) {
throw new IllegalArgumentException("No value specified for terms query");
}
this.fieldName = fieldName;
this.values = convertToBytesRefListIfStringList(values);
}
/**
* A filter for a field based on several terms matching on any of them.
*
* @param name The field name
* @param values The terms
*/
public TermsQueryBuilder(String name, Iterable values) {
this.name = name;
this.values = values;
public String fieldName() {
return this.fieldName;
}
public List<Object> values() {
return convertToStringListIfBytesRefList(this.values);
}
/**
@ -132,6 +162,10 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
return this;
}
public String minimumShouldMatch() {
return this.minimumShouldMatch;
}
/**
* Disables <tt>Similarity#coord(int,int)</tt> in scoring. Defaults to <tt>false</tt>.
* @deprecated use [bool] query instead
@ -142,72 +176,140 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
return this;
}
public boolean disableCoord() {
return this.disableCoord;
}
private boolean isTermsLookupQuery() {
return this.termsLookup != null;
}
public TermsQueryBuilder termsLookup(TermsLookup termsLookup) {
this.termsLookup = termsLookup;
return this;
}
public TermsLookup termsLookup() {
return this.termsLookup;
}
/**
* Sets the index name to lookup the terms from.
*/
public TermsQueryBuilder lookupIndex(String lookupIndex) {
this.lookupIndex = lookupIndex;
if (lookupIndex == null) {
throw new IllegalArgumentException("Lookup index cannot be set to null");
}
if (this.termsLookup == null) {
this.termsLookup = new TermsLookup();
}
this.termsLookup.index(lookupIndex);
return this;
}
/**
* Sets the index type to lookup the terms from.
* Sets the type name to lookup the terms from.
*/
public TermsQueryBuilder lookupType(String lookupType) {
this.lookupType = lookupType;
if (lookupType == null) {
throw new IllegalArgumentException("Lookup type cannot be set to null");
}
if (this.termsLookup == null) {
this.termsLookup = new TermsLookup();
}
this.termsLookup.type(lookupType);
return this;
}
/**
* Sets the doc id to lookup the terms from.
* Sets the document id to lookup the terms from.
*/
public TermsQueryBuilder lookupId(String lookupId) {
this.lookupId = lookupId;
if (lookupId == null) {
throw new IllegalArgumentException("Lookup id cannot be set to null");
}
if (this.termsLookup == null) {
this.termsLookup = new TermsLookup();
}
this.termsLookup.id(lookupId);
return this;
}
/**
* Sets the path within the document to lookup the terms from.
* Sets the path name to lookup the terms from.
*/
public TermsQueryBuilder lookupPath(String lookupPath) {
this.lookupPath = lookupPath;
if (lookupPath == null) {
throw new IllegalArgumentException("Lookup path cannot be set to null");
}
if (this.termsLookup == null) {
this.termsLookup = new TermsLookup();
}
this.termsLookup.path(lookupPath);
return this;
}
/**
* Sets the routing to lookup the terms from.
*/
public TermsQueryBuilder lookupRouting(String lookupRouting) {
this.lookupRouting = lookupRouting;
if (lookupRouting == null) {
throw new IllegalArgumentException("Lookup routing cannot be set to null");
}
if (this.termsLookup == null) {
this.termsLookup = new TermsLookup();
}
this.termsLookup.routing(lookupRouting);
return this;
}
/**
* Same as {@link #convertToBytesRefIfString} but on Iterable.
* @param objs the Iterable of input object
* @return the same input or a list of {@link BytesRef} representation if input was a list of type string
*/
private static List<Object> convertToBytesRefListIfStringList(Iterable<?> objs) {
if (objs == null) {
return null;
}
List<Object> newObjs = new ArrayList<>();
for (Object obj : objs) {
newObjs.add(convertToBytesRefIfString(obj));
}
return newObjs;
}
/**
* Same as {@link #convertToStringIfBytesRef} but on Iterable.
* @param objs the Iterable of input object
* @return the same input or a list of utf8 string if input was a list of type {@link BytesRef}
*/
private static List<Object> convertToStringListIfBytesRefList(Iterable<?> objs) {
if (objs == null) {
return null;
}
List<Object> newObjs = new ArrayList<>();
for (Object obj : objs) {
newObjs.add(convertToStringIfBytesRef(obj));
}
return newObjs;
}
@Override
public void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(NAME);
if (values == null) {
builder.startObject(name);
if (lookupIndex != null) {
builder.field("index", lookupIndex);
}
builder.field("type", lookupType);
builder.field("id", lookupId);
if (lookupRouting != null) {
builder.field("routing", lookupRouting);
}
builder.field("path", lookupPath);
if (isTermsLookupQuery()) {
builder.startObject(fieldName);
termsLookup.toXContent(builder, params);
builder.endObject();
} else {
builder.field(name, values);
builder.field(fieldName, convertToStringListIfBytesRefList(values));
}
if (minimumShouldMatch != null) {
builder.field("minimum_should_match", minimumShouldMatch);
}
if (disableCoord != null) {
builder.field("disable_coord", disableCoord);
}
builder.field("disable_coord", disableCoord);
printBoostAndQueryName(builder);
builder.endObject();
}
@ -215,4 +317,113 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
public String getWriteableName() {
return NAME;
}
@Override
protected Query doToQuery(QueryShardContext context) throws IOException {
List<Object> terms;
if (isTermsLookupQuery()) {
if (termsLookup.index() == null) {
termsLookup.index(context.index().name());
}
terms = context.indexQueryParserService().handleTermsLookup(termsLookup);
} else {
terms = values;
}
if (terms == null || terms.isEmpty()) {
return Queries.newMatchNoDocsQuery();
}
return handleTermsQuery(terms, fieldName, context, minimumShouldMatch, disableCoord);
}
private static Query handleTermsQuery(List<Object> terms, String fieldName, QueryShardContext context, String minimumShouldMatch, boolean disableCoord) {
MappedFieldType fieldType = context.fieldMapper(fieldName);
String indexFieldName;
if (fieldType != null) {
indexFieldName = fieldType.names().indexName();
} else {
indexFieldName = fieldName;
}
Query query;
if (context.isFilter()) {
if (fieldType != null) {
query = fieldType.termsQuery(terms, context);
} else {
BytesRef[] filterValues = new BytesRef[terms.size()];
for (int i = 0; i < filterValues.length; i++) {
filterValues[i] = BytesRefs.toBytesRef(terms.get(i));
}
query = new TermsQuery(indexFieldName, filterValues);
}
} else {
BooleanQuery bq = new BooleanQuery(disableCoord);
for (Object term : terms) {
if (fieldType != null) {
bq.add(fieldType.termQuery(term, context), BooleanClause.Occur.SHOULD);
} else {
bq.add(new TermQuery(new Term(indexFieldName, BytesRefs.toBytesRef(term))), BooleanClause.Occur.SHOULD);
}
}
Queries.applyMinimumShouldMatch(bq, minimumShouldMatch);
query = bq;
}
return query;
}
@Override
public QueryValidationException validate() {
QueryValidationException validationException = null;
if (this.fieldName == null) {
validationException = addValidationError("field name cannot be null.", validationException);
}
if (isTermsLookupQuery() && this.values != null) {
validationException = addValidationError("can't have both a terms query and a lookup query.", validationException);
}
if (isTermsLookupQuery()) {
QueryValidationException exception = termsLookup.validate();
if (exception != null) {
validationException = QueryValidationException.addValidationErrors(exception.validationErrors(), validationException);
}
}
return validationException;
}
@SuppressWarnings("unchecked")
@Override
protected TermsQueryBuilder doReadFrom(StreamInput in) throws IOException {
TermsQueryBuilder termsQueryBuilder = new TermsQueryBuilder(in.readString());
if (in.readBoolean()) {
termsQueryBuilder.termsLookup = TermsLookup.readTermsLookupFrom(in);
}
termsQueryBuilder.values = ((List<Object>) in.readGenericValue());
termsQueryBuilder.minimumShouldMatch = in.readOptionalString();
termsQueryBuilder.disableCoord = in.readBoolean();
return termsQueryBuilder;
}
@Override
protected void doWriteTo(StreamOutput out) throws IOException {
out.writeString(fieldName);
out.writeBoolean(isTermsLookupQuery());
if (isTermsLookupQuery()) {
termsLookup.writeTo(out);
}
out.writeGenericValue(values);
out.writeOptionalString(minimumShouldMatch);
out.writeBoolean(disableCoord);
}
@Override
protected int doHashCode() {
return Objects.hash(fieldName, values, minimumShouldMatch, disableCoord, termsLookup);
}
@Override
protected boolean doEquals(TermsQueryBuilder other) {
return Objects.equals(fieldName, other.fieldName) &&
Objects.equals(values, other.values) &&
Objects.equals(minimumShouldMatch, other.minimumShouldMatch) &&
Objects.equals(disableCoord, other.disableCoord) &&
Objects.equals(termsLookup, other.termsLookup);
}
}

View File

@ -20,39 +20,28 @@
package org.elasticsearch.index.query;
import com.google.common.collect.Lists;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.indices.cache.query.terms.TermsLookup;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.List;
/**
* Parser for terms query and terms lookup.
*
* Filters documents that have fields that match any of the provided terms (not analyzed)
*
* It also supports a terms lookup mechanism which can be used to fetch the term values from
* a document in an index.
*/
public class TermsQueryParser extends BaseQueryParserTemp {
public class TermsQueryParser extends BaseQueryParser {
private static final ParseField MIN_SHOULD_MATCH_FIELD = new ParseField("min_match", "min_should_match").withAllDeprecated("Use [bool] query instead");
private static final ParseField MIN_SHOULD_MATCH_FIELD = new ParseField("min_match", "min_should_match", "minimum_should_match")
.withAllDeprecated("Use [bool] query instead");
private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord").withAllDeprecated("Use [bool] query instead");
private static final ParseField EXECUTION_FIELD = new ParseField("execution").withAllDeprecated("execution is deprecated and has no effect");
private Client client;
@Inject
public TermsQueryParser() {
@ -63,32 +52,21 @@ public class TermsQueryParser extends BaseQueryParserTemp {
return new String[]{TermsQueryBuilder.NAME, "in"};
}
@Inject(optional = true)
public void setClient(Client client) {
this.client = client;
}
@Override
public Query parse(QueryShardContext context) throws IOException, QueryParsingException {
QueryParseContext parseContext = context.parseContext();
public QueryBuilder fromXContent(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
String queryName = null;
String currentFieldName = null;
String lookupIndex = parseContext.index().name();
String lookupType = null;
String lookupId = null;
String lookupPath = null;
String lookupRouting = null;
String fieldName = null;
List<Object> values = null;
String minShouldMatch = null;
boolean disableCoord = TermsQueryBuilder.DEFAULT_DISABLE_COORD;
TermsLookup termsLookup = null;
boolean disableCoord = false;
String queryName = null;
float boost = AbstractQueryBuilder.DEFAULT_BOOST;
XContentParser.Token token;
List<Object> terms = Lists.newArrayList();
String fieldName = null;
float boost = 1f;
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
@ -99,45 +77,10 @@ public class TermsQueryParser extends BaseQueryParserTemp {
throw new QueryParsingException(parseContext, "[terms] query does not support multiple fields");
}
fieldName = currentFieldName;
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
Object value = parser.objectBytes();
if (value == null) {
throw new QueryParsingException(parseContext, "No value specified for terms query");
}
terms.add(value);
}
values = parseValues(parseContext, parser);
} else if (token == XContentParser.Token.START_OBJECT) {
fieldName = currentFieldName;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if ("index".equals(currentFieldName)) {
lookupIndex = parser.text();
} else if ("type".equals(currentFieldName)) {
lookupType = parser.text();
} else if ("id".equals(currentFieldName)) {
lookupId = parser.text();
} else if ("path".equals(currentFieldName)) {
lookupPath = parser.text();
} else if ("routing".equals(currentFieldName)) {
lookupRouting = parser.textOrNull();
} else {
throw new QueryParsingException(parseContext, "[terms] query does not support [" + currentFieldName
+ "] within lookup element");
}
}
}
if (lookupType == null) {
throw new QueryParsingException(parseContext, "[terms] query lookup element requires specifying the type");
}
if (lookupId == null) {
throw new QueryParsingException(parseContext, "[terms] query lookup element requires specifying the id");
}
if (lookupPath == null) {
throw new QueryParsingException(parseContext, "[terms] query lookup element requires specifying the path");
}
termsLookup = parseTermsLookup(parseContext, parser);
} else if (token.isValue()) {
if (parseContext.parseFieldMatcher().match(currentFieldName, EXECUTION_FIELD)) {
// ignore
@ -159,58 +102,69 @@ public class TermsQueryParser extends BaseQueryParserTemp {
}
if (fieldName == null) {
throw new QueryParsingException(parseContext, "terms query requires a field name, followed by array of terms");
throw new QueryParsingException(parseContext, "terms query requires a field name, followed by array of terms or a document lookup specification");
}
MappedFieldType fieldType = context.fieldMapper(fieldName);
if (fieldType != null) {
fieldName = fieldType.names().indexName();
}
if (lookupId != null) {
final TermsLookup lookup = new TermsLookup(lookupIndex, lookupType, lookupId, lookupRouting, lookupPath, parseContext);
GetRequest getRequest = new GetRequest(lookup.getIndex(), lookup.getType(), lookup.getId()).preference("_local").routing(lookup.getRouting());
getRequest.copyContextAndHeadersFrom(SearchContext.current());
final GetResponse getResponse = client.get(getRequest).actionGet();
if (getResponse.isExists()) {
List<Object> values = XContentMapValues.extractRawValues(lookup.getPath(), getResponse.getSourceAsMap());
terms.addAll(values);
}
}
if (terms.isEmpty()) {
return Queries.newMatchNoDocsQuery();
}
Query query;
if (context.isFilter()) {
if (fieldType != null) {
query = fieldType.termsQuery(terms, context);
} else {
BytesRef[] filterValues = new BytesRef[terms.size()];
for (int i = 0; i < filterValues.length; i++) {
filterValues[i] = BytesRefs.toBytesRef(terms.get(i));
}
query = new TermsQuery(fieldName, filterValues);
}
TermsQueryBuilder termsQueryBuilder;
if (values == null) {
termsQueryBuilder = new TermsQueryBuilder(fieldName);
} else {
BooleanQuery bq = new BooleanQuery(disableCoord);
for (Object term : terms) {
if (fieldType != null) {
bq.add(fieldType.termQuery(term, context), Occur.SHOULD);
termsQueryBuilder = new TermsQueryBuilder(fieldName, values);
}
return termsQueryBuilder
.disableCoord(disableCoord)
.minimumShouldMatch(minShouldMatch)
.termsLookup(termsLookup)
.boost(boost)
.queryName(queryName);
}
private static List<Object> parseValues(QueryParseContext parseContext, XContentParser parser) throws IOException {
List<Object> values = Lists.newArrayList();
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
Object value = parser.objectBytes();
if (value == null) {
throw new QueryParsingException(parseContext, "No value specified for terms query");
}
values.add(value);
}
return values;
}
private static TermsLookup parseTermsLookup(QueryParseContext parseContext, XContentParser parser) throws IOException {
TermsLookup termsLookup = new TermsLookup();
XContentParser.Token token;
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if ("index".equals(currentFieldName)) {
termsLookup.index(parser.textOrNull());
} else if ("type".equals(currentFieldName)) {
termsLookup.type(parser.text());
} else if ("id".equals(currentFieldName)) {
termsLookup.id(parser.text());
} else if ("routing".equals(currentFieldName)) {
termsLookup.routing(parser.textOrNull());
} else if ("path".equals(currentFieldName)) {
termsLookup.path(parser.text());
} else {
bq.add(new TermQuery(new Term(fieldName, BytesRefs.toBytesRef(term))), Occur.SHOULD);
throw new QueryParsingException(parseContext, "[terms] query does not support [" + currentFieldName
+ "] within lookup element");
}
}
Queries.applyMinimumShouldMatch(bq, minShouldMatch);
query = bq;
}
query.setBoost(boost);
if (queryName != null) {
context.addNamedQuery(queryName, query);
if (termsLookup.type() == null) {
throw new QueryParsingException(parseContext, "[terms] query lookup element requires specifying the type");
}
return query;
if (termsLookup.id() == null) {
throw new QueryParsingException(parseContext, "[terms] query lookup element requires specifying the id");
}
if (termsLookup.path() == null) {
throw new QueryParsingException(parseContext, "[terms] query lookup element requires specifying the path");
}
return termsLookup;
}
@Override

View File

@ -0,0 +1,60 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.search.termslookup;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.indices.cache.query.terms.TermsLookup;
import org.elasticsearch.search.internal.SearchContext;
import java.util.ArrayList;
import java.util.List;
/**
* Service which retrieves terms from a {@link TermsLookup} specification
*/
public class TermsLookupFetchService extends AbstractComponent {
private final Client client;
@Inject
public TermsLookupFetchService(Client client, Settings settings) {
super(settings);
this.client = client;
}
public List<Object> fetch(TermsLookup termsLookup) {
List<Object> terms = new ArrayList<>();
GetRequest getRequest = new GetRequest(termsLookup.index(), termsLookup.type(), termsLookup.id())
.preference("_local").routing(termsLookup.routing());
getRequest.copyContextAndHeadersFrom(SearchContext.current());
final GetResponse getResponse = client.get(getRequest).actionGet();
if (getResponse.isExists()) {
List<Object> extractedValues = XContentMapValues.extractRawValues(termsLookup.path(), getResponse.getSourceAsMap());
terms.addAll(extractedValues);
}
return terms;
}
}

View File

@ -19,58 +19,162 @@
package org.elasticsearch.indices.cache.query.terms;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryValidationException;
import java.io.IOException;
import java.util.Objects;
/**
* Encapsulates the parameters needed to fetch terms.
*/
public class TermsLookup {
public class TermsLookup implements Writeable<TermsLookup>, ToXContent {
static final TermsLookup PROTOTYPE = new TermsLookup();
private final String index;
private final String type;
private final String id;
private final String routing;
private final String path;
private String index;
private String type;
private String id;
private String path;
private String routing;
@Nullable
private final QueryParseContext queryParseContext;
public TermsLookup() {
}
public TermsLookup(String index, String type, String id, String routing, String path, @Nullable QueryParseContext queryParseContext) {
public TermsLookup(String index, String type, String id, String path) {
this.index = index;
this.type = type;
this.id = id;
this.routing = routing;
this.path = path;
this.queryParseContext = queryParseContext;
}
public String getIndex() {
public String index() {
return index;
}
public String getType() {
public TermsLookup index(String index) {
this.index = index;
return this;
}
public String type() {
return type;
}
public String getId() {
public TermsLookup type(String type) {
this.type = type;
return this;
}
public String id() {
return id;
}
public String getRouting() {
return this.routing;
public TermsLookup id(String id) {
this.id = id;
return this;
}
public String getPath() {
public String path() {
return path;
}
@Nullable
public QueryParseContext getQueryParseContext() {
return queryParseContext;
public TermsLookup path(String path) {
this.path = path;
return this;
}
public String routing() {
return routing;
}
public TermsLookup routing(String routing) {
this.routing = routing;
return this;
}
@Override
public String toString() {
return index + "/" + type + "/" + id + "/" + path;
}
@Override
public TermsLookup readFrom(StreamInput in) throws IOException {
TermsLookup termsLookup = new TermsLookup();
termsLookup.index = in.readOptionalString();
termsLookup.type = in.readString();
termsLookup.id = in.readString();
termsLookup.path = in.readString();
termsLookup.routing = in.readOptionalString();
return termsLookup;
}
public static TermsLookup readTermsLookupFrom(StreamInput in) throws IOException {
return PROTOTYPE.readFrom(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(index);
out.writeString(type);
out.writeString(id);
out.writeString(path);
out.writeOptionalString(routing);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (index != null) {
builder.field("index", index);
}
builder.field("type", type);
builder.field("id", id);
builder.field("path", path);
if (routing != null) {
builder.field("routing", routing);
}
return builder;
}
@Override
public int hashCode() {
return Objects.hash(index, type, id, path, routing);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
TermsLookup other = (TermsLookup) obj;
return Objects.equals(index, other.index) &&
Objects.equals(type, other.type) &&
Objects.equals(id, other.id) &&
Objects.equals(path, other.path) &&
Objects.equals(routing, other.routing);
}
public QueryValidationException validate() {
QueryValidationException validationException = null;
if (id == null) {
validationException = addValidationError("[terms] query lookup element requires specifying the id.", validationException);
}
if (type == null) {
validationException = addValidationError("[terms] query lookup element requires specifying the type.", validationException);
}
if (path == null) {
validationException = addValidationError("[terms] query lookup element requires specifying the path.", validationException);
}
return validationException;
}
private static QueryValidationException addValidationError(String validationError, QueryValidationException validationException) {
return QueryValidationException.addValidationError("terms_lookup", validationError, validationException);
}
}

View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.query;
import com.google.common.collect.Lists;
import org.apache.lucene.search.Query;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@ -54,12 +55,14 @@ import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
import org.elasticsearch.index.query.support.QueryParsers;
import org.elasticsearch.index.search.termslookup.TermsLookupFetchService;
import org.elasticsearch.index.settings.IndexSettingsModule;
import org.elasticsearch.index.similarity.SimilarityModule;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.indices.cache.query.terms.TermsLookup;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.ESTestCase;
@ -73,7 +76,9 @@ import org.joda.time.DateTimeZone;
import org.junit.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.*;
@ -88,6 +93,8 @@ public abstract class BaseQueryTestCase<QB extends AbstractQueryBuilder<QB>> ext
protected static final String OBJECT_FIELD_NAME = "mapped_object";
protected static final String[] mappedFieldNames = new String[] { STRING_FIELD_NAME, INT_FIELD_NAME,
DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME };
protected static final String[] mappedFieldNamesSmall = new String[] { STRING_FIELD_NAME, INT_FIELD_NAME,
DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME };
private static Injector injector;
private static IndexQueryParserService queryParserService;
@ -169,6 +176,7 @@ public abstract class BaseQueryTestCase<QB extends AbstractQueryBuilder<QB>> ext
currentTypes[i] = type;
}
namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
queryParserService.setTermsLookupFetchService(new MockTermsLookupFetchService());
}
@AfterClass
@ -245,14 +253,18 @@ public abstract class BaseQueryTestCase<QB extends AbstractQueryBuilder<QB>> ext
* Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
*/
protected void assertParsedQuery(String queryAsString, QueryBuilder<?> expectedQuery) throws IOException {
QueryBuilder newQuery = parseQuery(queryAsString, expectedQuery);
assertNotSame(newQuery, expectedQuery);
assertEquals(expectedQuery, newQuery);
assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
}
protected QueryBuilder parseQuery(String queryAsString, QueryBuilder<?> expectedQuery) throws IOException {
XContentParser parser = XContentFactory.xContent(queryAsString).createParser(queryAsString);
QueryParseContext context = createParseContext();
context.reset(parser);
assertQueryHeader(parser, expectedQuery.getName());
QueryBuilder newQuery = queryParser(expectedQuery).fromXContent(context);
assertNotSame(newQuery, expectedQuery);
assertEquals(expectedQuery, newQuery);
assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
return queryParser(expectedQuery).fromXContent(context);
}
/**
@ -420,20 +432,52 @@ public abstract class BaseQueryTestCase<QB extends AbstractQueryBuilder<QB>> ext
/**
* create a random value for either {@link BaseQueryTestCase#BOOLEAN_FIELD_NAME}, {@link BaseQueryTestCase#INT_FIELD_NAME},
* {@link BaseQueryTestCase#DOUBLE_FIELD_NAME} or {@link BaseQueryTestCase#STRING_FIELD_NAME}, or a String value by default
* {@link BaseQueryTestCase#DOUBLE_FIELD_NAME}, {@link BaseQueryTestCase#STRING_FIELD_NAME} or
* {@link BaseQueryTestCase#DATE_FIELD_NAME}, or a String value by default
*/
protected static Object randomValueForField(String fieldName) {
protected static Object getRandomValueForFieldName(String fieldName) {
Object value;
switch (fieldName) {
case BOOLEAN_FIELD_NAME: value = randomBoolean(); break;
case INT_FIELD_NAME: value = randomInt(); break;
case DOUBLE_FIELD_NAME: value = randomDouble(); break;
case STRING_FIELD_NAME: value = randomAsciiOfLengthBetween(1, 10); break;
default : value = randomAsciiOfLengthBetween(1, 10);
case STRING_FIELD_NAME:
value = rarely() ? randomUnicodeOfLength(10) : randomAsciiOfLengthBetween(1, 10); // unicode in 10% cases
break;
case INT_FIELD_NAME:
value = randomIntBetween(0, 10);
break;
case DOUBLE_FIELD_NAME:
value = randomDouble() * 10;
break;
case BOOLEAN_FIELD_NAME:
value = randomBoolean();
break;
case DATE_FIELD_NAME:
value = new DateTime(System.currentTimeMillis(), DateTimeZone.UTC).toString();
break;
default:
value = randomAsciiOfLengthBetween(1, 10);
}
return value;
}
/**
* Helper method to return a mapped or a random field
*/
protected String getRandomFieldName() {
// if no type is set then return a random field name
if (currentTypes == null || currentTypes.length == 0 || randomBoolean()) {
return randomAsciiOfLengthBetween(1, 10);
}
return randomFrom(mappedFieldNamesSmall);
}
/**
* Helper method to return a random field (mapped or unmapped) and a value
*/
protected Tuple<String, Object> getRandomFieldNameAndValue() {
String fieldName = getRandomFieldName();
return new Tuple<>(fieldName, getRandomValueForFieldName(fieldName));
}
/**
* Helper method to return a random rewrite method
*/
@ -473,42 +517,6 @@ public abstract class BaseQueryTestCase<QB extends AbstractQueryBuilder<QB>> ext
return (currentTypes.length == 0) ? MetaData.ALL : randomFrom(currentTypes);
}
/**
* Helper method to return a random field (mapped or unmapped) and a value
*/
protected static Tuple<String, Object> getRandomFieldNameAndValue() {
// if no type is set then return random field name and value
if (currentTypes == null || currentTypes.length == 0) {
return new Tuple<String, Object>(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 50));
}
// mapped fields
String fieldName = randomFrom(mappedFieldNames);
Object value = randomAsciiOfLengthBetween(1, 50);
switch(fieldName) {
case STRING_FIELD_NAME:
value = rarely() ? randomUnicodeOfLength(10) : value; // unicode in 10% cases
break;
case INT_FIELD_NAME:
value = randomIntBetween(0, 10);
break;
case DOUBLE_FIELD_NAME:
value = randomDouble() * 10;
break;
case BOOLEAN_FIELD_NAME:
value = randomBoolean();
break;
case DATE_FIELD_NAME:
value = new DateTime(System.currentTimeMillis(), DateTimeZone.UTC).toString();
break;
} // all other fields assigned to random string
// unmapped fields
if (randomBoolean()) {
fieldName = randomAsciiOfLengthBetween(1, 10);
}
return new Tuple<>(fieldName, value);
}
protected static Fuzziness randomFuzziness(String fieldName) {
Fuzziness fuzziness = Fuzziness.AUTO;
switch (fieldName) {
@ -531,4 +539,29 @@ public abstract class BaseQueryTestCase<QB extends AbstractQueryBuilder<QB>> ext
protected static boolean isNumericFieldName(String fieldName) {
return INT_FIELD_NAME.equals(fieldName) || DOUBLE_FIELD_NAME.equals(fieldName);
}
protected static class MockTermsLookupFetchService extends TermsLookupFetchService {
private static List<Object> randomTerms = new ArrayList<>();
public MockTermsLookupFetchService() {
super(null, Settings.Builder.EMPTY_SETTINGS);
String[] strings = generateRandomStringArray(10, 10, false, true);
for (String string : strings) {
randomTerms.add(string);
if (rarely()) {
randomTerms.add(null);
}
}
}
@Override
public List<Object> fetch(TermsLookup termsLookup) {
return randomTerms;
}
public static List<Object> getRandomTerms() {
return randomTerms;
}
}
}

View File

@ -36,7 +36,7 @@ public class FuzzyQueryBuilderTest extends BaseQueryTestCase<FuzzyQueryBuilder>
@Override
protected FuzzyQueryBuilder doCreateTestQueryBuilder() {
Tuple<String, Object> fieldAndValue = getRandomFieldNameAndValue();
Tuple<String, Object> fieldAndValue = getRandomFieldNameAndValue();
FuzzyQueryBuilder query = new FuzzyQueryBuilder(fieldAndValue.v1(), fieldAndValue.v2());
if (randomBoolean()) {
query.fuzziness(randomFuzziness(query.fieldName()));

View File

@ -61,7 +61,7 @@ public class SpanTermQueryBuilderTest extends BaseTermQueryTestCase<SpanTermQuer
clauses[0] = first;
for (int i = 1; i < amount; i++) {
// we need same field name in all clauses, so we only randomize value
SpanTermQueryBuilder spanTermQuery = new SpanTermQueryBuilder(first.fieldName(), randomValueForField(first.fieldName()));
SpanTermQueryBuilder spanTermQuery = new SpanTermQueryBuilder(first.fieldName(), getRandomValueForFieldName(first.fieldName()));
if (randomBoolean()) {
spanTermQuery.boost(2.0f / randomIntBetween(1, 20));
}

View File

@ -0,0 +1,190 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.indices.cache.query.terms.TermsLookup;
import org.hamcrest.Matchers;
import org.junit.Test;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import static org.hamcrest.Matchers.*;
public class TermsQueryBuilderTest extends BaseQueryTestCase<TermsQueryBuilder> {
@Override
protected TermsQueryBuilder doCreateTestQueryBuilder() {
TermsQueryBuilder query;
// terms query or lookup query
if (randomBoolean()) {
// make between 0 and 5 different values of the same type
String fieldName = getRandomFieldName();
Object[] values = new Object[randomInt(5)];
for (int i = 0; i < values.length; i++) {
values[i] = getRandomValueForFieldName(fieldName);
}
query = new TermsQueryBuilder(fieldName, values);
} else {
// right now the mock service returns us a list of strings
query = new TermsQueryBuilder(randomBoolean() ? randomAsciiOfLengthBetween(1,10) : STRING_FIELD_NAME);
query.termsLookup(randomTermsLookup());
}
if (randomBoolean()) {
query.minimumShouldMatch(randomInt(100) + "%");
}
if (randomBoolean()) {
query.disableCoord(randomBoolean());
}
return query;
}
private TermsLookup randomTermsLookup() {
return new TermsLookup(
randomBoolean() ? randomAsciiOfLength(10) : null,
randomAsciiOfLength(10),
randomAsciiOfLength(10),
randomAsciiOfLength(10)
).routing(randomBoolean() ? randomAsciiOfLength(10) : null);
}
@Override
protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
assertThat(query, instanceOf(BooleanQuery.class));
BooleanQuery booleanQuery = (BooleanQuery) query;
// we only do the check below for string fields (otherwise we'd have to decode the values)
if (!queryBuilder.fieldName().equals(STRING_FIELD_NAME) && queryBuilder.termsLookup() == null) {
return;
}
// expected returned terms depending on whether we have a terms query or a terms lookup query
List<Object> terms;
if (queryBuilder.termsLookup() != null) {
terms = MockTermsLookupFetchService.getRandomTerms();
} else {
terms = queryBuilder.values();
}
// compare whether we have the expected list of terms returned
Iterator<Object> iter = terms.iterator();
for (BooleanClause booleanClause : booleanQuery) {
assertThat(booleanClause.getQuery(), instanceOf(TermQuery.class));
Term term = ((TermQuery) booleanClause.getQuery()).getTerm();
Object next = iter.next();
if (next == null) {
continue;
}
assertThat(term, equalTo(new Term(queryBuilder.fieldName(), next.toString())));
}
}
@Test
public void testValidate() {
TermsQueryBuilder termsQueryBuilder = new TermsQueryBuilder(null, "term");
assertThat(termsQueryBuilder.validate().validationErrors().size(), is(1));
termsQueryBuilder = new TermsQueryBuilder("field", "term").termsLookup(randomTermsLookup());
assertThat(termsQueryBuilder.validate().validationErrors().size(), is(1));
termsQueryBuilder = new TermsQueryBuilder(null, "term").termsLookup(randomTermsLookup());
assertThat(termsQueryBuilder.validate().validationErrors().size(), is(2));
termsQueryBuilder = new TermsQueryBuilder("field", "term");
assertNull(termsQueryBuilder.validate());
}
@Test
public void testValidateLookupQuery() {
TermsQueryBuilder termsQuery = new TermsQueryBuilder("field").termsLookup(new TermsLookup());
int totalExpectedErrors = 3;
if (randomBoolean()) {
termsQuery.lookupId("id");
totalExpectedErrors--;
}
if (randomBoolean()) {
termsQuery.lookupType("type");
totalExpectedErrors--;
}
if (randomBoolean()) {
termsQuery.lookupPath("path");
totalExpectedErrors--;
}
assertValidate(termsQuery, totalExpectedErrors);
}
@Test
public void testNullValues() {
try {
switch (randomInt(6)) {
case 0:
new TermsQueryBuilder("field", (String) null);
break;
case 1:
new TermsQueryBuilder("field", (int[]) null);
break;
case 2:
new TermsQueryBuilder("field", (long[]) null);
break;
case 3:
new TermsQueryBuilder("field", (float[]) null);
break;
case 4:
new TermsQueryBuilder("field", (double[]) null);
break;
case 5:
new TermsQueryBuilder("field", (Object) null);
break;
default:
new TermsQueryBuilder("field", (Iterable) null);
break;
}
fail("should have failed with IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), Matchers.containsString("No value specified for terms query"));
}
}
@Test
public void testBothValuesAndLookupSet() throws IOException {
String query = "{\n" +
" \"terms\": {\n" +
" \"field\": [\n" +
" \"blue\",\n" +
" \"pill\"\n" +
" ],\n" +
" \"field_lookup\": {\n" +
" \"index\": \"pills\",\n" +
" \"type\": \"red\",\n" +
" \"id\": \"3\",\n" +
" \"path\": \"white rabbit\"\n" +
" }\n" +
" }\n" +
"}";
QueryBuilder termsQueryBuilder = parseQuery(query, TermsQueryBuilder.PROTOTYPE);
assertThat(termsQueryBuilder.validate().validationErrors().size(), is(1));
}
}

View File

@ -59,6 +59,11 @@ Removed `wrapperQueryBuilder(byte[] source, int offset, int length)`. Instead si
use `wrapperQueryBuilder(byte[] source)`. Updated the static factory methods in
QueryBuilders accordingly.
==== TermsQuery with TermsLookup
Removed `getIndex()`, `getType()`, `getId()`, `getPath()`, `getRouting()` in favor of
`index()`, `type()`, `id()`, `path()` and `routing()`.
==== Operator
Removed the enums called `Operator` from `MatchQueryBuilder`, `QueryStringQueryBuilder`,