honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Percolator should cache index field data instances. 

Before the index reader used by the percolator didn't allow to register a CoreCloseListener, but now it does, making it safe to cache index field data cache entries.
Creating field data structures is relatively expensive and caching them can save a lot of noise if many queries are evaluated in a percolator call.

Closes #6806
Closes #7081
This commit is contained in:
Martijn van Groningen 2014-07-29 19:13:03 +02:00
parent 3c83c0f9d7
commit c8cc59df57
4 changed files with 19 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
View File

@ -38,11 +38,10 @@ import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import java.util.ArrayList;
import java.util.Collection;
@ -299,40 +298,4 @@ public class IndexFieldDataService extends AbstractIndexComponent {
return (IFD) fieldData;
}
public <IFD extends IndexFieldData<?>> IFD getForFieldDirect(FieldMapper<?> mapper) {
final FieldMapper.Names fieldNames = mapper.names();
final FieldDataType type = mapper.fieldDataType();
if (type == null) {
throw new ElasticsearchIllegalArgumentException("found no fielddata type for field [" + fieldNames.fullName() + "]");
}
final boolean docValues = mapper.hasDocValues();
IndexFieldData.Builder builder = null;
String format = type.getFormat(indexSettings);
if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) {
logger.warn("field [" + fieldNames.fullName() + "] has no doc values, will use default field data format");
format = null;
}
if (format != null) {
builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), format));
if (builder == null) {
logger.warn("failed to find format [" + format + "] for field [" + fieldNames.fullName() + "], will use default");
}
}
if (builder == null && docValues) {
builder = docValuesBuildersByType.get(type.getType());
}
if (builder == null) {
builder = buildersByType.get(type.getType());
}
if (builder == null) {
throw new ElasticsearchIllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType());
}
CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
@SuppressWarnings("unchecked")
IFD ifd = (IFD) builder.build(index, indexSettings, mapper, new IndexFieldDataCache.None(), circuitBreakerService, indexService.mapperService());
return ifd;
}
}

16
src/main/java/org/elasticsearch/index/query/QueryParseContext.java
View File

@ -88,17 +88,17 @@ public class QueryParseContext {
private EnumSet<ParseField.Flag> parseFlags = ParseField.EMPTY_FLAGS;
private final boolean disableCaching;
private final boolean disableFilterCaching;
public QueryParseContext(Index index, IndexQueryParserService indexQueryParser) {
this(index, indexQueryParser, false);
}
public QueryParseContext(Index index, IndexQueryParserService indexQueryParser, boolean disableCaching) {
public QueryParseContext(Index index, IndexQueryParserService indexQueryParser, boolean disableFilterCaching) {
this.index = index;
this.indexQueryParser = indexQueryParser;
this.propagateNoCache = disableCaching;
this.disableCaching = disableCaching;
this.propagateNoCache = disableFilterCaching;
this.disableFilterCaching = disableFilterCaching;
}
public void parseFlags(EnumSet<ParseField.Flag> parseFlags) {
@ -178,7 +178,7 @@ public class QueryParseContext {
if (filter == null) {
return null;
}
if (this.disableCaching || this.propagateNoCache || filter instanceof NoCacheFilter) {
if (this.disableFilterCaching || this.propagateNoCache || filter instanceof NoCacheFilter) {
return filter;
}
if (cacheKey != null) {
@ -188,11 +188,7 @@ public class QueryParseContext {
}
public <IFD extends IndexFieldData<?>> IFD getForField(FieldMapper<?> mapper) {
if (disableCaching) {
return indexQueryParser.fieldDataService.getForFieldDirect(mapper);
} else {
return indexQueryParser.fieldDataService.getForField(mapper);
}
return indexQueryParser.fieldDataService.getForField(mapper);
}
public void addNamedFilter(String name, Filter filter) {

5
src/test/java/org/elasticsearch/test/ExternalTestCluster.java
View File

@ -120,13 +120,16 @@ public final class ExternalTestCluster extends TestCluster {
public void ensureEstimatedStats() {
if (size() > 0) {
NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats()
.clear().setBreaker(true).execute().actionGet();
.clear().setBreaker(true).setIndices(true).execute().actionGet();
for (NodeStats stats : nodeStats.getNodes()) {
assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(),
stats.getBreaker().getStats(CircuitBreaker.Name.FIELDDATA).getEstimated(), equalTo(0L));
// ExternalTestCluster does not check the request breaker,
// because checking it requires a network request, which in
// turn increments the breaker, making it non-0
assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat("Filter cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
}
}
}

8
src/test/java/org/elasticsearch/test/InternalTestCluster.java
View File

@ -32,6 +32,8 @@ import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.elasticsearch.cache.recycler.PageCacheRecyclerModule;
import org.elasticsearch.client.Client;
@ -70,6 +72,7 @@ import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.internal.InternalNode;
import org.elasticsearch.node.service.NodeService;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.test.cache.recycler.MockBigArraysModule;
@ -1539,6 +1542,11 @@ public final class InternalTestCluster extends TestCluster {
} catch (Exception e) {
fail("Exception during check for request breaker reset to 0: " + e);
}
NodeService nodeService = getInstanceFromNode(NodeService.class, nodeAndClient.node);
NodeStats stats = nodeService.stats(CommonStatsFlags.ALL, false, false, false, false, false, false, false, false, false);
assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
assertThat("Filter cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
}
}
}