honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Query DSL: Add ids query and filter to fetch docs by ids (do not require _id field to be indexed), closes #865.

This commit is contained in:
kimchy 2011-04-20 00:30:05 +03:00
parent be2a456cc0
commit c3f3c268c8
9 changed files with 500 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/IndexQueryParserModule.java
View File

@ -223,6 +223,7 @@ public class IndexQueryParserModule extends AbstractModule {
bindings.processXContentQueryParser(HasChildQueryParser.NAME, HasChildQueryParser.class);
bindings.processXContentQueryParser(TopChildrenQueryParser.NAME, TopChildrenQueryParser.class);
bindings.processXContentQueryParser(DisMaxQueryParser.NAME, DisMaxQueryParser.class);
bindings.processXContentQueryParser(IdsQueryParser.NAME, IdsQueryParser.class);
bindings.processXContentQueryParser(MatchAllQueryParser.NAME, MatchAllQueryParser.class);
bindings.processXContentQueryParser(QueryStringQueryParser.NAME, QueryStringQueryParser.class);
bindings.processXContentQueryParser(BoostingQueryParser.NAME, BoostingQueryParser.class);
@ -251,6 +252,7 @@ public class IndexQueryParserModule extends AbstractModule {
@Override public void processXContentFilterParsers(XContentFilterParsersBindings bindings) {
bindings.processXContentQueryFilter(HasChildFilterParser.NAME, HasChildFilterParser.class);
bindings.processXContentQueryFilter(IdsFilterParser.NAME, IdsFilterParser.class);
bindings.processXContentQueryFilter(TermFilterParser.NAME, TermFilterParser.class);
bindings.processXContentQueryFilter(TermsFilterParser.NAME, TermsFilterParser.class);
bindings.processXContentQueryFilter(RangeFilterParser.NAME, RangeFilterParser.class);

9
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/FilterBuilders.java
View File

@ -33,6 +33,15 @@ public abstract class FilterBuilders {
return new MatchAllFilterBuilder();
}
/**
* Creates a new ids filter with the provided doc/mapping type.
*
* @param type The type
*/
public static IdsFilterBuilder idsFilter(String type) {
return new IdsFilterBuilder(type);
}
/**
* A filter for a field based on a term.
*

85
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/IdsFilterBuilder.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query.xcontent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* A filter that will return only documents matching specific ids (and a type).
*/
public class IdsFilterBuilder extends BaseFilterBuilder {
private String type;
private List<String> values = new ArrayList<String>();
private String filterName;
/**
* Create an ids filter based on the type.
*/
public IdsFilterBuilder(String type) {
this.type = type;
}
/**
* Adds ids to the filter.
*/
public IdsFilterBuilder addIds(String... ids) {
values.addAll(Arrays.asList(ids));
return this;
}
/**
* Adds ids to the filter.
*/
public IdsFilterBuilder ids(String... ids) {
return addIds(ids);
}
/**
* Sets the filter name for the filter that can be used when searching for matched_filters per hit.
*/
public IdsFilterBuilder filterName(String filterName) {
this.filterName = filterName;
return this;
}
@Override public void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(IdsFilterParser.NAME);
builder.field("type", type);
builder.startArray("values");
for (Object value : values) {
builder.value(value);
}
builder.endArray();
if (filterName != null) {
builder.field("_name", filterName);
}
builder.endObject();
}
}

91
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/IdsFilterParser.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query.xcontent;
import org.apache.lucene.search.Filter;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.query.QueryParsingException;
import org.elasticsearch.index.search.UidFilter;
import org.elasticsearch.index.settings.IndexSettings;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class IdsFilterParser extends AbstractIndexComponent implements XContentFilterParser {
public static final String NAME = "ids";
@Inject public IdsFilterParser(Index index, @IndexSettings Settings settings) {
super(index, settings);
}
@Override public String[] names() {
return new String[]{NAME};
}
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
List<String> ids = new ArrayList<String>();
String type = null;
String filterName = null;
String currentFieldName = null;
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_ARRAY) {
if ("values".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String value = parser.textOrNull();
if (value == null) {
throw new QueryParsingException(index, "No value specified for term filter");
}
ids.add(value);
}
}
} else if (token.isValue()) {
if ("type".equals(currentFieldName) || "_type".equals(currentFieldName)) {
type = parser.text();
} else if ("_name".equals(currentFieldName)) {
filterName = parser.text();
}
}
}
if (type == null) {
throw new QueryParsingException(index, "[ids] filter, no type provided");
}
if (ids.size() == 0) {
throw new QueryParsingException(index, "[ids] filter, no ids values provided");
}
UidFilter filter = new UidFilter(type, ids, parseContext.indexCache().bloomCache());
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);
}
return filter;
}
}

81
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/IdsQueryBuilder.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query.xcontent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* A query that will return only documents matching specific ids (and a type).
*/
public class IdsQueryBuilder extends BaseQueryBuilder {
private String type;
private List<String> values = new ArrayList<String>();
private float boost = -1;
public IdsQueryBuilder(String type) {
this.type = type;
}
/**
* Adds ids to the filter.
*/
public IdsQueryBuilder addIds(String... ids) {
values.addAll(Arrays.asList(ids));
return this;
}
/**
* Adds ids to the filter.
*/
public IdsQueryBuilder ids(String... ids) {
return addIds(ids);
}
/**
* Sets the boost for this query. Documents matching this query will (in addition to the normal
* weightings) have their score multiplied by the boost provided.
*/
public IdsQueryBuilder boost(float boost) {
this.boost = boost;
return this;
}
@Override protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(IdsQueryParser.NAME);
builder.field("type", type);
builder.startArray("values");
for (Object value : values) {
builder.value(value);
}
builder.endArray();
if (boost != -1) {
builder.field("boost", boost);
}
builder.endObject();
}
}

96
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/IdsQueryParser.java Normal file
View File

@ -0,0 +1,96 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query.xcontent;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.query.QueryParsingException;
import org.elasticsearch.index.search.UidFilter;
import org.elasticsearch.index.settings.IndexSettings;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
*
*/
public class IdsQueryParser extends AbstractIndexComponent implements XContentQueryParser {
public static final String NAME = "ids";
@Inject public IdsQueryParser(Index index, @IndexSettings Settings settings) {
super(index, settings);
}
@Override public String[] names() {
return new String[]{NAME};
}
@Override public Query parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
List<String> ids = new ArrayList<String>();
String type = null;
String currentFieldName = null;
float boost = 1.0f;
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_ARRAY) {
if ("values".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String value = parser.textOrNull();
if (value == null) {
throw new QueryParsingException(index, "No value specified for term filter");
}
ids.add(value);
}
}
} else if (token.isValue()) {
if ("type".equals(currentFieldName) || "_type".equals(currentFieldName)) {
type = parser.text();
} else if ("boost".equals(currentFieldName)) {
boost = parser.floatValue();
}
}
}
if (type == null) {
throw new QueryParsingException(index, "[ids] query, no type provided");
}
if (ids.size() == 0) {
throw new QueryParsingException(index, "[ids] query, no ids values provided");
}
UidFilter filter = new UidFilter(type, ids, parseContext.indexCache().bloomCache());
// no need for constant score filter, since we don't cache the filter, and it always takes deletes into account
ConstantScoreQuery query = new ConstantScoreQuery(filter);
query.setBoost(boost);
return query;
}
}

9
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/QueryBuilders.java
View File

@ -42,6 +42,15 @@ public abstract class QueryBuilders {
return new DisMaxQueryBuilder();
}
/**
* Constructs a query that will match only specific ids within a type.
*
* @param type The mapping/doc type
*/
public static IdsQueryBuilder idsQuery(String type) {
return new IdsQueryBuilder(type);
}
/**
* A Query that matches documents containing a term.
*

97
modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.bloom.BloomFilter;
import org.elasticsearch.index.cache.bloom.BloomCache;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.UidFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
public class UidFilter extends Filter {
private final Term[] uids;
private final BloomCache bloomCache;
public UidFilter(String type, List<String> ids, BloomCache bloomCache) {
this.bloomCache = bloomCache;
uids = new Term[ids.size()];
for (int i = 0; i < ids.size(); i++) {
uids[i] = new Term(UidFieldMapper.NAME, Uid.createUid(type, ids.get(i)));
}
Arrays.sort(uids);
}
@Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
BloomFilter filter = bloomCache.filter(reader, UidFieldMapper.NAME, true);
OpenBitSet set = null;
TermDocs td = null;
try {
for (Term uid : uids) {
UnicodeUtil.UTF8Result utf8 = Unicode.fromStringAsUtf8(uid.text());
if (!filter.isPresent(utf8.result, 0, utf8.length)) {
continue;
}
if (td == null) {
td = reader.termDocs();
}
td.seek(uid);
while (td.next()) {
if (set == null) {
set = new OpenBitSet(reader.maxDoc());
}
set.fastSet(td.doc());
}
}
} finally {
if (td != null) {
td.close();
}
}
return set;
}
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
UidFilter uidFilter = (UidFilter) o;
if (!Arrays.equals(uids, uidFilter.uids)) return false;
return true;
}
@Override public int hashCode() {
return uids != null ? Arrays.hashCode(uids) : 0;
}
}

30
modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java
View File

@ -97,6 +97,36 @@ public class SimpleQueryTests extends AbstractNodesTests {
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
}
@Test public void idsFilterTests() {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1)).execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("field1", "value2").execute().actionGet();
client.prepareIndex("test", "type1", "3").setSource("field1", "value3").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse = client.prepareSearch().setQuery(constantScoreQuery(idsFilter("type1").ids("1", "3"))).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("3")));
assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("1"), equalTo("3")));
searchResponse = client.prepareSearch().setQuery(idsQuery("type1").ids("1", "3")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("3")));
assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("1"), equalTo("3")));
searchResponse = client.prepareSearch().setQuery(idsQuery("type1").ids("7", "10")).execute().actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(0l));
}
@Test public void filterExistsMissingTests() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();