honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Query DSL: Fine grained filter caching control with sensible defaults, closes #424.

This commit is contained in:
kimchy 2010-10-13 01:37:43 +02:00
parent 3fa214653e
commit 8a8a4d648a
22 changed files with 154 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/dictionaries/kimchy.xml
View File

@ -12,6 +12,7 @@
<w>banon</w>
<w>bindhost</w>
<w>birthdate</w>
<w>bitset</w>
<w>blobstore</w>
<w>bool</w>
<w>booleans</w>

16
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/geo/GeoBoundingBoxFilter.java
View File

@ -68,6 +68,14 @@ public class GeoBoundingBoxFilter extends Filter {
//checks to see if bounding box crosses 180 degrees
if (topLeft.lon > bottomRight.lon) {
return new GetDocSet(reader.maxDoc()) {
@Override public boolean isCacheable() {
// not cacheable for several reasons:
// 1. It is only relevant when _cache is set to true, and then, we really want to create in mem bitset
// 2. Its already fast without in mem bitset, since it works with field data
return false;
}
@Override public boolean get(int doc) throws IOException {
if (!fieldData.hasValue(doc)) {
return false;
@ -107,6 +115,14 @@ public class GeoBoundingBoxFilter extends Filter {
};
} else {
return new GetDocSet(reader.maxDoc()) {
@Override public boolean isCacheable() {
// not cacheable for several reasons:
// 1. It is only relevant when _cache is set to true, and then, we really want to create in mem bitset
// 2. Its already fast without in mem bitset, since it works with field data
return false;
}
@Override public boolean get(int doc) throws IOException {
if (!fieldData.hasValue(doc)) {
return false;

4
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/geo/GeoDistanceFilter.java
View File

@ -80,7 +80,11 @@ public class GeoDistanceFilter extends Filter {
@Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, reader, fieldName);
return new GetDocSet(reader.maxDoc()) {
@Override public boolean isCacheable() {
// not cacheable for several reasons:
// 1. It is only relevant when _cache is set to true, and then, we really want to create in mem bitset
// 2. Its already fast without in mem bitset, since it works with field data
return false;
}

8
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/geo/GeoPolygonFilter.java
View File

@ -59,6 +59,14 @@ public class GeoPolygonFilter extends Filter {
final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, reader, fieldName);
return new GetDocSet(reader.maxDoc()) {
@Override public boolean isCacheable() {
// not cacheable for several reasons:
// 1. It is only relevant when _cache is set to true, and then, we really want to create in mem bitset
// 2. Its already fast without in mem bitset, since it works with field data
return false;
}
@Override public boolean get(int doc) throws IOException {
if (!fieldData.hasValue(doc)) {
return false;

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/geo/GeoPointFieldMapper.java
View File

@ -76,7 +76,7 @@ public class GeoPointFieldMapper implements XContentMapper, ArrayValueMapperPars
private Integer precisionStep;
private int precision = 16; // to get high precision
private int precision = GeoHashUtils.PRECISION;
private Field.Store store = Defaults.STORE;

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java
View File

@ -48,7 +48,7 @@ public final class QueryParsers {
return query;
}
DocumentMapper docMapper = smartFieldMappers.docMapper();
return new FilteredQuery(query, parseContext.cacheFilterIfPossible(docMapper.typeFilter()));
return new FilteredQuery(query, parseContext.cacheFilter(docMapper.typeFilter()));
}
public static Filter wrapSmartNameFilter(Filter filter, @Nullable MapperService.SmartNameFieldMappers smartFieldMappers,
@ -60,6 +60,6 @@ public final class QueryParsers {
return filter;
}
DocumentMapper docMapper = smartFieldMappers.docMapper();
return new AndFilter(ImmutableList.of(parseContext.cacheFilterIfPossible(docMapper.typeFilter()), filter));
return new AndFilter(ImmutableList.of(parseContext.cacheFilter(docMapper.typeFilter()), filter));
}
}

12
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/AndFilterParser.java
View File

@ -54,7 +54,7 @@ public class AndFilterParser extends AbstractIndexComponent implements XContentF
ArrayList<Filter> filters = newArrayList();
boolean cache = true;
boolean cache = false;
String filterName = null;
String currentFieldName = null;
@ -91,13 +91,11 @@ public class AndFilterParser extends AbstractIndexComponent implements XContentF
throw new QueryParsingException(index, "[or] filter requires 'filters' to be set on it'");
}
if (cache) {
for (int i = 0; i < filters.size(); i++) {
filters.set(i, parseContext.cacheFilterIfPossible(filters.get(i)));
}
}
// no need to cache this one
AndFilter filter = new AndFilter(filters);
Filter filter = new AndFilter(filters);
if (cache) {
filter = parseContext.cacheFilter(filter);
}
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);
}

16
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/BoolFilterParser.java
View File

@ -56,7 +56,7 @@ public class BoolFilterParser extends AbstractIndexComponent implements XContent
List<OpenFilterClause> clauses = newArrayList();
boolean cache = true;
boolean cache = false;
String filterName = null;
String currentFieldName = null;
@ -95,16 +95,14 @@ public class BoolFilterParser extends AbstractIndexComponent implements XContent
}
}
XBooleanFilter filter = new XBooleanFilter();
XBooleanFilter boolFilter = new XBooleanFilter();
for (OpenFilterClause filterClause : clauses) {
if (cache) {
filterClause.setFilter(parseContext.cacheFilterIfPossible(filterClause.getFilter()));
}
filter.add(filterClause);
boolFilter.add(filterClause);
}
Filter filter = boolFilter;
if (cache) {
filter = parseContext.cacheFilter(filter);
}
// no need to cache this one, inner queries will be cached and that's is good enough (I think...)
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);
}

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ConstantScoreQueryParser.java
View File

@ -81,7 +81,7 @@ public class ConstantScoreQueryParser extends AbstractIndexComponent implements
Query query;
if (cache) {
Filter nonCachedFilter = filter;
filter = parseContext.cacheFilterIfPossible(filter);
filter = parseContext.cacheFilter(filter);
if (parseContext.indexEngine().readerClonedOnDeletion() && (filter != nonCachedFilter)) {
query = new DeletionAwareConstantScoreQuery(filter, true);
} else {

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/FQueryFilterParser.java
View File

@ -54,6 +54,7 @@ public class FQueryFilterParser extends AbstractIndexComponent implements XConte
XContentParser parser = parseContext.parser();
Query query = null;
boolean cache = false;
String filterName = null;
String currentFieldName = null;
@ -68,10 +69,15 @@ public class FQueryFilterParser extends AbstractIndexComponent implements XConte
} else if (token.isValue()) {
if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
}
}
}
Filter filter = new QueryWrapperFilter(query);
if (cache) {
filter = parseContext.cacheFilter(filter);
}
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);
}

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/FilteredQueryParser.java
View File

@ -83,7 +83,7 @@ public class FilteredQueryParser extends AbstractIndexComponent implements XCont
// cache if required
if (cache) {
filter = parseContext.cacheFilterIfPossible(filter);
filter = parseContext.cacheFilter(filter);
}
// we don't cache the filter, we assume it is already cached in the filter parsers...

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/GeoBoundingBoxFilterParser.java
View File

@ -56,6 +56,7 @@ public class GeoBoundingBoxFilterParser extends AbstractIndexComponent implement
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
boolean cache = false;
String fieldName = null;
GeoBoundingBoxFilter.Point topLeft = new GeoBoundingBoxFilter.Point();
GeoBoundingBoxFilter.Point bottomRight = new GeoBoundingBoxFilter.Point();
@ -143,6 +144,8 @@ public class GeoBoundingBoxFilterParser extends AbstractIndexComponent implement
} else if (token.isValue()) {
if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
}
}
}
@ -160,6 +163,9 @@ public class GeoBoundingBoxFilterParser extends AbstractIndexComponent implement
Filter filter = new GeoBoundingBoxFilter(topLeft, bottomRight, fieldName, parseContext.indexCache().fieldData());
if (cache) {
filter = parseContext.cacheFilter(filter);
}
filter = wrapSmartNameFilter(filter, parseContext.smartFieldMappers(fieldName), parseContext);
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/GeoDistanceFilterParser.java
View File

@ -67,6 +67,7 @@ public class GeoDistanceFilterParser extends AbstractIndexComponent implements X
XContentParser.Token token;
boolean cache = false;
String filterName = null;
String currentFieldName = null;
double lat = 0;
@ -130,6 +131,8 @@ public class GeoDistanceFilterParser extends AbstractIndexComponent implements X
fieldName = currentFieldName.substring(0, currentFieldName.length() - GeoPointFieldMapper.Names.GEOHASH_SUFFIX.length());
} else if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
} else {
// assume the value is the actual value
String value = parser.text();
@ -162,6 +165,9 @@ public class GeoDistanceFilterParser extends AbstractIndexComponent implements X
fieldName = mapper.names().indexName();
Filter filter = new GeoDistanceFilter(lat, lon, distance, geoDistance, fieldName, parseContext.indexCache().fieldData());
if (cache) {
filter = parseContext.cacheFilter(filter);
}
filter = wrapSmartNameFilter(filter, parseContext.smartFieldMappers(fieldName), parseContext);
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/GeoPolygonFilterParser.java
View File

@ -69,6 +69,7 @@ public class GeoPolygonFilterParser extends AbstractIndexComponent implements XC
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
boolean cache = false;
String fieldName = null;
List<GeoPolygonFilter.Point> points = Lists.newArrayList();
@ -140,6 +141,8 @@ public class GeoPolygonFilterParser extends AbstractIndexComponent implements XC
} else if (token.isValue()) {
if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
}
}
}
@ -159,6 +162,9 @@ public class GeoPolygonFilterParser extends AbstractIndexComponent implements XC
fieldName = mapper.names().indexName();
Filter filter = new GeoPolygonFilter(points.toArray(new GeoPolygonFilter.Point[points.size()]), fieldName, parseContext.indexCache().fieldData());
if (cache) {
filter = parseContext.cacheFilter(filter);
}
filter = wrapSmartNameFilter(filter, parseContext.smartFieldMappers(fieldName), parseContext);
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);

7
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/NotFilterParser.java
View File

@ -50,7 +50,7 @@ public class NotFilterParser extends AbstractIndexComponent implements XContentF
XContentParser parser = parseContext.parser();
Filter filter = null;
boolean cache = true;
boolean cache = false;
String filterName = null;
String currentFieldName = null;
@ -75,11 +75,10 @@ public class NotFilterParser extends AbstractIndexComponent implements XContentF
throw new QueryParsingException(index, "filter is required when using `not` filter");
}
Filter notFilter = new NotFilter(filter);
if (cache) {
filter = parseContext.cacheFilterIfPossible(filter);
notFilter = parseContext.cacheFilter(notFilter);
}
// no need to cache this one
NotFilter notFilter = new NotFilter(filter);
if (filterName != null) {
parseContext.addNamedFilter(filterName, notFilter);
}

12
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/OrFilterParser.java
View File

@ -54,7 +54,7 @@ public class OrFilterParser extends AbstractIndexComponent implements XContentFi
ArrayList<Filter> filters = newArrayList();
boolean cache = true;
boolean cache = false;
String filterName = null;
String currentFieldName = null;
@ -91,13 +91,11 @@ public class OrFilterParser extends AbstractIndexComponent implements XContentFi
throw new QueryParsingException(index, "[or] filter requires 'filters' to be set on it'");
}
if (cache) {
for (int i = 0; i < filters.size(); i++) {
filters.set(i, parseContext.cacheFilterIfPossible(filters.get(i)));
}
}
// no need to cache this one
OrFilter filter = new OrFilter(filters);
Filter filter = new OrFilter(filters);
if (cache) {
filter = parseContext.cacheFilter(filter);
}
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/PrefixFilterParser.java
View File

@ -53,6 +53,7 @@ public class PrefixFilterParser extends AbstractIndexComponent implements XConte
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
boolean cache = true; // default to true, make sense to cache prefix
String fieldName = null;
String value = null;
@ -65,6 +66,8 @@ public class PrefixFilterParser extends AbstractIndexComponent implements XConte
} else if (token.isValue()) {
if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
} else {
fieldName = currentFieldName;
value = parser.text();
@ -85,6 +88,9 @@ public class PrefixFilterParser extends AbstractIndexComponent implements XConte
}
Filter filter = new PrefixFilter(new Term(fieldName, value));
if (cache) {
filter = parseContext.cacheFilter(filter);
}
filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/QueryParseContext.java
View File

@ -108,7 +108,7 @@ public class QueryParseContext {
return multiFieldQueryParser;
}
public Filter cacheFilterIfPossible(Filter filter) {
public Filter cacheFilter(Filter filter) {
return indexQueryParser.indexCache.filter().cache(filter);
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/RangeFilterParser.java
View File

@ -52,6 +52,7 @@ public class RangeFilterParser extends AbstractIndexComponent implements XConten
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
boolean cache = true; // default to true, usually same range are used, I think....
String fieldName = null;
String from = null;
String to = null;
@ -96,6 +97,8 @@ public class RangeFilterParser extends AbstractIndexComponent implements XConten
} else if (token.isValue()) {
if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
}
}
}
@ -110,6 +113,9 @@ public class RangeFilterParser extends AbstractIndexComponent implements XConten
if (filter == null) {
filter = new TermRangeFilter(fieldName, from, to, includeLower, includeUpper);
}
if (cache) {
filter = parseContext.cacheFilter(filter);
}
filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);

86
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ScriptFilterParser.java
View File

@ -30,8 +30,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryParsingException;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.script.ScriptService;
@ -61,6 +59,8 @@ public class ScriptFilterParser extends AbstractIndexComponent implements XConte
XContentParser.Token token;
boolean cache = false; // no need to cache it by default, changes a lot?
// also, when caching, since its isCacheable is false, will result in loading all bit set...
String script = null;
String scriptLang = null;
Map<String, Object> params = null;
@ -81,6 +81,8 @@ public class ScriptFilterParser extends AbstractIndexComponent implements XConte
scriptLang = parser.text();
} else if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
}
}
}
@ -92,7 +94,10 @@ public class ScriptFilterParser extends AbstractIndexComponent implements XConte
params = Maps.newHashMap();
}
Filter filter = new ScriptFilter(scriptLang, script, params, parseContext.mapperService(), parseContext.indexCache().fieldData(), parseContext.scriptService());
Filter filter = new ScriptFilter(scriptLang, script, params, parseContext.scriptService());
if (cache) {
filter = parseContext.cacheFilter(filter);
}
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);
}
@ -101,26 +106,22 @@ public class ScriptFilterParser extends AbstractIndexComponent implements XConte
public static class ScriptFilter extends Filter {
private final String scriptLang;
private final String script;
private final Map<String, Object> params;
private final MapperService mapperService;
private final SearchScript searchScript;
private final FieldDataCache fieldDataCache;
private final ScriptService scriptService;
private ScriptFilter(String scriptLang, String script, Map<String, Object> params,
MapperService mapperService, FieldDataCache fieldDataCache, ScriptService scriptService) {
this.scriptLang = scriptLang;
private ScriptFilter(String scriptLang, String script, Map<String, Object> params, ScriptService scriptService) {
this.script = script;
this.params = params;
this.mapperService = mapperService;
this.fieldDataCache = fieldDataCache;
this.scriptService = scriptService;
SearchContext context = SearchContext.current();
if (context == null) {
throw new ElasticSearchIllegalStateException("No search context on going...");
}
this.searchScript = new SearchScript(context.scriptSearchLookup(), scriptLang, script, params, scriptService);
}
@Override public String toString() {
@ -150,31 +151,40 @@ public class ScriptFilterParser extends AbstractIndexComponent implements XConte
}
@Override public DocIdSet getDocIdSet(final IndexReader reader) throws IOException {
SearchContext context = SearchContext.current();
if (context == null) {
throw new ElasticSearchIllegalStateException("No search context on going...");
}
final SearchScript searchScript = new SearchScript(context.scriptSearchLookup(), scriptLang, script, params, scriptService);
searchScript.setNextReader(reader);
return new GetDocSet(reader.maxDoc()) {
@Override public boolean isCacheable() {
return false; // though it is, we want to cache it into in memory rep so it will be faster
}
return new ScriptDocSet(reader, searchScript);
}
@Override public boolean get(int doc) throws IOException {
Object val = searchScript.execute(doc, params);
if (val == null) {
return false;
}
if (val instanceof Boolean) {
return (Boolean) val;
}
if (val instanceof Number) {
return ((Number) val).longValue() != 0;
}
throw new IOException("Can't handle type [" + val + "] in script filter");
static class ScriptDocSet extends GetDocSet {
private final SearchScript searchScript;
public ScriptDocSet(IndexReader reader, SearchScript searchScript) {
super(reader.maxDoc());
this.searchScript = searchScript;
}
@Override public boolean isCacheable() {
// not cacheable for several reasons:
// 1. The script service is shared and holds the current reader executing against, and it
// gets changed on each getDocIdSet (which is fine for sequential reader search)
// 2. If its really going to be cached (the _cache setting), its better to just load it into in memory bitset
return false;
}
@Override public boolean get(int doc) throws IOException {
Object val = searchScript.execute(doc);
if (val == null) {
return false;
}
};
if (val instanceof Boolean) {
return (Boolean) val;
}
if (val instanceof Number) {
return ((Number) val).longValue() != 0;
}
throw new IOException("Can't handle type [" + val + "] in script filter");
}
}
}
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/TermFilterParser.java
View File

@ -53,6 +53,7 @@ public class TermFilterParser extends AbstractIndexComponent implements XContent
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
boolean cache = true; // we want to cache term by default
String fieldName = null;
String value = null;
@ -65,6 +66,8 @@ public class TermFilterParser extends AbstractIndexComponent implements XContent
} else if (token.isValue()) {
if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
} else {
fieldName = currentFieldName;
value = parser.text();
@ -86,6 +89,9 @@ public class TermFilterParser extends AbstractIndexComponent implements XContent
if (filter == null) {
filter = new TermFilter(new Term(fieldName, value));
}
if (cache) {
filter = parseContext.cacheFilter(filter);
}
filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);

10
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/TermsFilterParser.java
View File

@ -56,6 +56,7 @@ public class TermsFilterParser extends AbstractIndexComponent implements XConten
XContentParser parser = parseContext.parser();
MapperService.SmartNameFieldMappers smartNameFieldMappers = null;
boolean cache = false;
TermsFilter termsFilter = new PublicTermsFilter();
String filterName = null;
String currentFieldName = null;
@ -87,11 +88,18 @@ public class TermsFilterParser extends AbstractIndexComponent implements XConten
} else if (token.isValue()) {
if ("_name".equals(currentFieldName)) {
filterName = parser.text();
} else if ("_cache".equals(currentFieldName)) {
cache = parser.booleanValue();
}
}
}
Filter filter = wrapSmartNameFilter(termsFilter, smartNameFieldMappers, parseContext);
Filter filter = termsFilter;
if (cache) {
filter = parseContext.cacheFilter(filter);
}
filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
if (filterName != null) {
parseContext.addNamedFilter(filterName, filter);
}