honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Better support with _type is marked as not indexed, allowing to filter by type, closes #866.

This commit is contained in:
kimchy 2011-04-20 01:31:38 +03:00
parent c3f3c268c8
commit 3b21759bec
18 changed files with 262 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/elasticsearch/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java
View File

@ -121,7 +121,7 @@ public class MapperQueryParser extends QueryParser {
if (currentMapper != null) {
Query query = null;
if (currentMapper.useFieldQueryWithQueryString()) {
query = currentMapper.fieldQuery(queryText);
query = currentMapper.fieldQuery(queryText, parseContext);
}
if (query == null) {
query = super.getFieldQuery(currentMapper.names().indexName(), queryText, quoted);

3
modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.MetaDataMappingService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.TypeFieldMapper;
import org.elasticsearch.index.query.xcontent.FilterBuilders;
import org.elasticsearch.index.query.xcontent.QueryBuilders;
import org.elasticsearch.threadpool.ThreadPool;
@ -96,7 +95,7 @@ public class TransportDeleteMappingAction extends TransportMasterNodeOperationAc
final AtomicReference<Throwable> failureRef = new AtomicReference<Throwable>();
final CountDownLatch latch = new CountDownLatch(1);
deleteByQueryAction.execute(Requests.deleteByQueryRequest(request.indices()).query(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(TypeFieldMapper.NAME, request.type()))), new ActionListener<DeleteByQueryResponse>() {
deleteByQueryAction.execute(Requests.deleteByQueryRequest(request.indices()).query(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.typeFilter(request.type()))), new ActionListener<DeleteByQueryResponse>() {
@Override public void onResponse(DeleteByQueryResponse deleteByQueryResponse) {
refreshAction.execute(Requests.refreshRequest(request.indices()), new ActionListener<RefreshResponse>() {
@Override public void onResponse(RefreshResponse refreshResponse) {

5
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View File

@ -28,6 +28,7 @@ import org.apache.lucene.search.Query;
import org.elasticsearch.common.util.concurrent.Immutable;
import org.elasticsearch.common.util.concurrent.ThreadSafe;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.query.xcontent.QueryParseContext;
/**
* @author kimchy (shay.banon)
@ -140,7 +141,7 @@ public interface FieldMapper<T> {
String indexedValue(String value);
/**
* Should the field query {@link #fieldQuery(String)} be used when detecting this
* Should the field query {@link #fieldQuery(String, org.elasticsearch.index.query.xcontent.QueryParseContext)} be used when detecting this
* field in query string.
*/
boolean useFieldQueryWithQueryString();
@ -148,7 +149,7 @@ public interface FieldMapper<T> {
/**
* A field query for the specified value.
*/
Query fieldQuery(String value);
Query fieldQuery(String value, QueryParseContext context);
/**
* A term query to use when parsing a query string. Can return <tt>null</tt>.

29
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -23,7 +23,9 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilterClause;
import org.apache.lucene.search.PublicTermsFilter;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableMap;
@ -32,6 +34,7 @@ import org.elasticsearch.common.collect.UnmodifiableIterator;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.lucene.search.TermFilter;
import org.elasticsearch.common.lucene.search.XBooleanFilter;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadSafe;
@ -259,11 +262,37 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
}
return docMapper.typeFilter();
}
// see if we can use terms filter
boolean useTermsFilter = true;
for (String type : types) {
DocumentMapper docMapper = documentMapper(type);
if (docMapper == null) {
useTermsFilter = false;
break;
}
if (!docMapper.typeMapper().indexed()) {
useTermsFilter = false;
break;
}
}
if (useTermsFilter) {
PublicTermsFilter termsFilter = new PublicTermsFilter();
for (String type : types) {
termsFilter.addTerm(new Term(TypeFieldMapper.NAME, type));
}
return termsFilter;
} else {
XBooleanFilter bool = new XBooleanFilter();
for (String type : types) {
DocumentMapper docMapper = documentMapper(type);
if (docMapper == null) {
bool.add(new FilterClause(new TermFilter(new Term(TypeFieldMapper.NAME, type)), BooleanClause.Occur.SHOULD));
} else {
bool.add(new FilterClause(docMapper.typeFilter(), BooleanClause.Occur.SHOULD));
}
}
return bool;
}
}
/**

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/Uid.java
View File

@ -65,6 +65,10 @@ public final class Uid {
return type + DELIMITER + id;
}
public static String typePrefix(String type) {
return type + DELIMITER;
}
public static Uid createUid(String uid) {
int delimiterIndex = uid.indexOf(DELIMITER); // type is not allowed to have # in it..., ids can
return new Uid(uid.substring(0, delimiterIndex), uid.substring(delimiterIndex + 1));

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/AbstractFieldMapper.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapperListener;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.query.xcontent.QueryParseContext;
import java.io.IOException;
@ -316,7 +317,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, XContent
return false;
}
@Override public Query fieldQuery(String value) {
@Override public Query fieldQuery(String value, QueryParseContext context) {
return new TermQuery(new Term(names.indexName(), indexedValue(value)));
}

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/AllFieldMapper.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.common.lucene.all.AllTermQuery;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.query.xcontent.QueryParseContext;
import java.io.IOException;
@ -106,7 +107,7 @@ public class AllFieldMapper extends AbstractFieldMapper<Void> implements org.ela
return new AllTermQuery(term);
}
@Override public Query fieldQuery(String value) {
@Override public Query fieldQuery(String value, QueryParseContext context) {
return new AllTermQuery(new Term(names.indexName(), value));
}

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/NumberFieldMapper.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.query.xcontent.QueryParseContext;
import java.io.Reader;
@ -123,7 +124,7 @@ public abstract class NumberFieldMapper<T extends Number> extends AbstractFieldM
* Numeric field level query are basically range queries with same value and included. That's the recommended
* way to execute it.
*/
@Override public Query fieldQuery(String value) {
@Override public Query fieldQuery(String value, QueryParseContext context) {
return rangeQuery(value, value, true, true);
}

23
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/TypeFieldMapper.java
View File

@ -23,9 +23,17 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DeletionAwareConstantScoreQuery;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.PrefixFilter;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.search.TermFilter;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.UidFieldMapper;
import org.elasticsearch.index.query.xcontent.QueryParseContext;
import java.io.IOException;
@ -101,6 +109,21 @@ public class TypeFieldMapper extends AbstractFieldMapper<String> implements org.
return new Term(names.indexName(), value);
}
@Override public Filter fieldFilter(String value) {
if (index == Field.Index.NO) {
return new PrefixFilter(new Term(UidFieldMapper.NAME, Uid.typePrefix(value)));
}
return new TermFilter(new Term(names.indexName(), value));
}
@Override public Query fieldQuery(String value, QueryParseContext context) {
return new DeletionAwareConstantScoreQuery(context.cacheFilter(fieldFilter(value)));
}
@Override public boolean useFieldQueryWithQueryString() {
return true;
}
@Override protected Field parseCreateField(ParseContext context) throws IOException {
if (index == Field.Index.NO && store == Field.Store.NO) {
return null;

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java
View File

@ -30,7 +30,6 @@ import org.elasticsearch.common.compress.lzf.LZF;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.CachedStreamInput;
import org.elasticsearch.common.io.stream.LZFStreamInput;
import org.elasticsearch.common.lucene.search.TermFilter;
import org.elasticsearch.common.thread.ThreadLocals;
import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.index.analysis.NamedAnalyzer;
@ -258,7 +257,7 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
this.indexAnalyzer = indexAnalyzer;
this.searchAnalyzer = searchAnalyzer;
this.typeFilter = new TermFilter(typeMapper().term(type));
this.typeFilter = typeMapper().fieldFilter(type);
rootObjectMapper.putMapper(idFieldMapper);
if (boostFieldMapper != null) {

1
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/IndexQueryParserModule.java
View File

@ -252,6 +252,7 @@ public class IndexQueryParserModule extends AbstractModule {
@Override public void processXContentFilterParsers(XContentFilterParsersBindings bindings) {
bindings.processXContentQueryFilter(HasChildFilterParser.NAME, HasChildFilterParser.class);
bindings.processXContentQueryFilter(TypeFilterParser.NAME, TypeFilterParser.class);
bindings.processXContentQueryFilter(IdsFilterParser.NAME, IdsFilterParser.class);
bindings.processXContentQueryFilter(TermFilterParser.NAME, TermFilterParser.class);
bindings.processXContentQueryFilter(TermsFilterParser.NAME, TermsFilterParser.class);

7
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/FilterBuilders.java
View File

@ -42,6 +42,13 @@ public abstract class FilterBuilders {
return new IdsFilterBuilder(type);
}
/**
* A filter based on doc/mapping type.
*/
public static TypeFilterBuilder typeFilter(String type) {
return new TypeFilterBuilder(type);
}
/**
* A filter for a field based on a term.
*

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/TermQueryParser.java
View File

@ -90,7 +90,7 @@ public class TermQueryParser extends AbstractIndexComponent implements XContentQ
MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName);
if (smartNameFieldMappers != null) {
if (smartNameFieldMappers.hasMapper()) {
query = smartNameFieldMappers.mapper().fieldQuery(value);
query = smartNameFieldMappers.mapper().fieldQuery(value, parseContext);
}
}
if (query == null) {

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/TermsQueryParser.java
View File

@ -108,7 +108,7 @@ public class TermsQueryParser extends AbstractIndexComponent implements XContent
BooleanQuery query = new BooleanQuery(disableCoord);
for (String value : values) {
if (mapper != null) {
query.add(new BooleanClause(mapper.fieldQuery(value), BooleanClause.Occur.SHOULD));
query.add(new BooleanClause(mapper.fieldQuery(value, parseContext), BooleanClause.Occur.SHOULD));
} else {
query.add(new TermQuery(new Term(fieldName, value)), BooleanClause.Occur.SHOULD);
}

39
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/TypeFilterBuilder.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query.xcontent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
public class TypeFilterBuilder extends BaseFilterBuilder {
private final String type;
public TypeFilterBuilder(String type) {
this.type = type;
}
@Override protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(TypeFilterParser.NAME);
builder.field("value", type);
builder.endObject();
}
}

77
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/TypeFilterParser.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.query.xcontent;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Filter;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.TermFilter;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.TypeFieldMapper;
import org.elasticsearch.index.query.QueryParsingException;
import org.elasticsearch.index.settings.IndexSettings;
import java.io.IOException;
public class TypeFilterParser extends AbstractIndexComponent implements XContentFilterParser {
public static final String NAME = "type";
@Inject public TypeFilterParser(Index index, @IndexSettings Settings settings) {
super(index, settings);
}
@Override public String[] names() {
return new String[]{NAME};
}
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
XContentParser.Token token = parser.nextToken();
if (token != XContentParser.Token.FIELD_NAME) {
throw new QueryParsingException(index, "type filter should have a value field, and the type name");
}
String fieldName = parser.currentName();
if (!fieldName.equals("value")) {
throw new QueryParsingException(index, "type filter should have a value field, and the type name");
}
token = parser.nextToken();
if (token != XContentParser.Token.VALUE_STRING) {
throw new QueryParsingException(index, "type filter should have a value field, and the type name");
}
String type = parser.text();
// move to the next token
parser.nextToken();
Filter filter;
DocumentMapper documentMapper = parseContext.mapperService().documentMapper(type);
if (documentMapper == null) {
filter = new TermFilter(new Term(TypeFieldMapper.NAME, type));
} else {
filter = documentMapper.typeFilter();
}
return parseContext.cacheFilter(filter);
}
}

14
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/lucene/DoubleIndexingDocTest.java
View File

@ -45,25 +45,25 @@ public class DoubleIndexingDocTest {
IndexReader reader = writer.getReader();
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs topDocs = searcher.search(mapper.mappers().smartName("field1").mapper().fieldQuery("value1"), 10);
TopDocs topDocs = searcher.search(mapper.mappers().smartName("field1").mapper().fieldQuery("value1", null), 10);
assertThat(topDocs.totalHits, equalTo(2));
topDocs = searcher.search(mapper.mappers().smartName("field2").mapper().fieldQuery("1"), 10);
topDocs = searcher.search(mapper.mappers().smartName("field2").mapper().fieldQuery("1", null), 10);
assertThat(topDocs.totalHits, equalTo(2));
topDocs = searcher.search(mapper.mappers().smartName("field3").mapper().fieldQuery("1.1"), 10);
topDocs = searcher.search(mapper.mappers().smartName("field3").mapper().fieldQuery("1.1", null), 10);
assertThat(topDocs.totalHits, equalTo(2));
topDocs = searcher.search(mapper.mappers().smartName("field4").mapper().fieldQuery("2010-01-01"), 10);
topDocs = searcher.search(mapper.mappers().smartName("field4").mapper().fieldQuery("2010-01-01", null), 10);
assertThat(topDocs.totalHits, equalTo(2));
topDocs = searcher.search(mapper.mappers().smartName("field5").mapper().fieldQuery("1"), 10);
topDocs = searcher.search(mapper.mappers().smartName("field5").mapper().fieldQuery("1", null), 10);
assertThat(topDocs.totalHits, equalTo(2));
topDocs = searcher.search(mapper.mappers().smartName("field5").mapper().fieldQuery("2"), 10);
topDocs = searcher.search(mapper.mappers().smartName("field5").mapper().fieldQuery("2", null), 10);
assertThat(topDocs.totalHits, equalTo(2));
topDocs = searcher.search(mapper.mappers().smartName("field5").mapper().fieldQuery("3"), 10);
topDocs = searcher.search(mapper.mappers().smartName("field5").mapper().fieldQuery("3", null), 10);
assertThat(topDocs.totalHits, equalTo(2));
}
}

60
modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java
View File

@ -27,6 +27,7 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.elasticsearch.common.xcontent.XContentFactory.*;
import static org.elasticsearch.index.query.xcontent.FilterBuilders.*;
import static org.elasticsearch.index.query.xcontent.QueryBuilders.*;
import static org.hamcrest.MatcherAssert.*;
@ -97,14 +98,69 @@ public class SimpleQueryTests extends AbstractNodesTests {
assertThat(searchResponse.hits().totalHits(), equalTo(1l));
}
@Test public void idsFilterTests() {
@Test public void typeFilterTypeIndexedTests() throws Exception {
typeFilterTests("not_analyzed");
}
@Test public void typeFilterTypeNotIndexedTests() throws Exception {
typeFilterTests("no");
}
private void typeFilterTests(String index) throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1)).execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1))
.addMapping("type1", jsonBuilder().startObject().startObject("type1")
.startObject("_type").field("index", index).endObject()
.endObject().endObject())
.addMapping("type2", jsonBuilder().startObject().startObject("type2")
.startObject("_type").field("index", index).endObject()
.endObject().endObject())
.execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet();
client.prepareIndex("test", "type2", "1").setSource("field1", "value1").execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "2").setSource("field1", "value1").execute().actionGet();
client.prepareIndex("test", "type2", "2").setSource("field1", "value1").execute().actionGet();
client.prepareIndex("test", "type2", "3").setSource("field1", "value1").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
assertThat(client.prepareCount().setQuery(filteredQuery(matchAllQuery(), typeFilter("type1"))).execute().actionGet().count(), equalTo(2l));
assertThat(client.prepareCount().setQuery(filteredQuery(matchAllQuery(), typeFilter("type2"))).execute().actionGet().count(), equalTo(3l));
assertThat(client.prepareCount().setTypes("type1").setQuery(matchAllQuery()).execute().actionGet().count(), equalTo(2l));
assertThat(client.prepareCount().setTypes("type2").setQuery(matchAllQuery()).execute().actionGet().count(), equalTo(3l));
assertThat(client.prepareCount().setTypes("type1", "type2").setQuery(matchAllQuery()).execute().actionGet().count(), equalTo(5l));
}
@Test public void idsFilterTestsIdIndexed() throws Exception {
idsFilterTests("not_analyzed");
}
@Test public void idsFilterTestsIdNotIndexed() throws Exception {
idsFilterTests("no");
}
private void idsFilterTests(String index) throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1))
.addMapping("type1", jsonBuilder().startObject().startObject("type1")
.startObject("_id").field("index", index).endObject()
.endObject().endObject())
.execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();