honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve on shard level filter/id cache stats 

use just the removal listener and back to the IndexReader#coreCacheKey as the actual field as part of the cache key
This commit is contained in:
Shay Banon 2013-04-06 00:02:42 +02:00
parent 815917fbf8
commit 9f6c8c88f3
7 changed files with 64 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/org/elasticsearch/common/lucene/docset/DocIdSets.java
View File

@ -33,6 +33,14 @@ import java.io.IOException;
*/
public class DocIdSets {
public static long sizeInBytes(DocIdSet docIdSet) {
if (docIdSet instanceof FixedBitSet) {
return ((FixedBitSet) docIdSet).getBits().length * 8 + 16;
}
// only for empty ones and unknowns...
return 1;
}
/**
* Is it an empty {@link DocIdSet}?
*/

17
src/main/java/org/elasticsearch/index/cache/filter/ShardFilterCache.java vendored
View File

@ -19,7 +19,11 @@
package org.elasticsearch.index.cache.filter;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import org.apache.lucene.search.DocIdSet;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.docset.DocIdSets;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
@ -29,7 +33,7 @@ import org.elasticsearch.index.shard.ShardId;
/**
*/
public class ShardFilterCache extends AbstractIndexShardComponent {
public class ShardFilterCache extends AbstractIndexShardComponent implements RemovalListener<WeightedFilterCache.FilterCacheKey, DocIdSet> {
final CounterMetric evictionsMetric = new CounterMetric();
final CounterMetric totalMetric = new CounterMetric();
@ -43,14 +47,17 @@ public class ShardFilterCache extends AbstractIndexShardComponent {
return new FilterCacheStats(totalMetric.count(), evictionsMetric.count());
}
public void onCached(WeightedFilterCache.FilterCacheKey cacheKey, long sizeInBytes) {
public void onCached(long sizeInBytes) {
totalMetric.inc(sizeInBytes);
}
public void onRemoval(WeightedFilterCache.FilterCacheKey cacheKey, boolean evicted, long sizeInBytes) {
if (evicted) {
@Override
public void onRemoval(RemovalNotification<WeightedFilterCache.FilterCacheKey, DocIdSet> removalNotification) {
if (removalNotification.wasEvicted()) {
evictionsMetric.inc();
}
totalMetric.dec(sizeInBytes);
if (removalNotification.getValue() != null) {
totalMetric.dec(DocIdSets.sizeInBytes(removalNotification.getValue()));
}
}
}

82
src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java vendored
View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.cache.filter.weighted;
import com.google.common.cache.Cache;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.common.cache.Weigher;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
@ -29,8 +28,8 @@ import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.docset.DocIdSets;
import org.elasticsearch.common.lucene.search.CachedFilter;
@ -51,12 +50,12 @@ import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
import java.io.IOException;
import java.util.concurrent.ConcurrentMap;
public class WeightedFilterCache extends AbstractIndexComponent implements FilterCache, SegmentReader.CoreClosedListener, RemovalListener<WeightedFilterCache.FilterCacheKey, DocIdSet> {
public class WeightedFilterCache extends AbstractIndexComponent implements FilterCache, SegmentReader.CoreClosedListener {
final IndicesFilterCache indicesFilterCache;
IndexService indexService;
final ConcurrentMap<IndexReader, Boolean> seenReaders = ConcurrentCollections.newConcurrentMap();
final ConcurrentMap<Object, Boolean> seenReaders = ConcurrentCollections.newConcurrentMap();
@Inject
public WeightedFilterCache(Index index, @IndexSettings Settings indexSettings, IndicesFilterCache indicesFilterCache) {
@ -95,8 +94,8 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte
public void clear(String reason, String[] keys) {
logger.debug("clear keys [], reason [{}]", reason, keys);
for (String key : keys) {
for (IndexReader reader : seenReaders.keySet()) {
indicesFilterCache.cache().invalidate(new FilterCacheKey(this, reader, new CacheKeyFilter.Key(key)));
for (Object readerKey : seenReaders.keySet()) {
indicesFilterCache.cache().invalidate(new FilterCacheKey(readerKey, new CacheKeyFilter.Key(key)));
}
}
}
@ -146,13 +145,13 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte
if (filter instanceof CacheKeyFilter) {
filterKey = ((CacheKeyFilter) filter).cacheKey();
}
FilterCacheKey cacheKey = new FilterCacheKey(this.cache, context.reader(), filterKey);
FilterCacheKey cacheKey = new FilterCacheKey(context.reader().getCoreCacheKey(), filterKey);
Cache<FilterCacheKey, DocIdSet> innerCache = cache.indicesFilterCache.cache();
DocIdSet cacheValue = innerCache.getIfPresent(cacheKey);
if (cacheValue == null) {
if (!cache.seenReaders.containsKey(context.reader().getCoreCacheKey())) {
Boolean previous = cache.seenReaders.putIfAbsent(context.reader(), Boolean.TRUE);
Boolean previous = cache.seenReaders.putIfAbsent(context.reader().getCoreCacheKey(), Boolean.TRUE);
if (previous == null) {
// we add a core closed listener only, for non core IndexReaders we rely on clear being called (percolator for example)
if (context.reader() instanceof SegmentReader) {
@ -165,8 +164,15 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte
cacheValue = DocIdSets.toCacheable(context.reader(), filter.getDocIdSet(context, context.reader().getLiveDocs()));
// we might put the same one concurrently, that's fine, it will be replaced and the removal
// will be called
ShardId shardId = ShardUtils.extractShardId(context.reader());
if (shardId != null) {
IndexShard shard = cache.indexService.shard(shardId.id());
if (shard != null) {
cacheKey.removalListener = shard.filterCache();
shard.filterCache().onCached(DocIdSets.sizeInBytes(cacheValue));
}
}
innerCache.put(cacheKey, cacheValue);
cache.onCached(cacheKey, cacheValue);
}
// note, we don't wrap the return value with a BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs) because
@ -194,65 +200,27 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte
@Override
public int weigh(FilterCacheKey key, DocIdSet value) {
int weight = (int) Math.min(sizeInBytes(value), Integer.MAX_VALUE);
int weight = (int) Math.min(DocIdSets.sizeInBytes(value), Integer.MAX_VALUE);
return weight == 0 ? 1 : weight;
}
}
// this will only be called for our index / data, IndicesFilterCache makes sure it works like this based on the
// index we register the listener with
@Override
public void onRemoval(RemovalNotification<FilterCacheKey, DocIdSet> removalNotification) {
if (removalNotification.getKey() != null && removalNotification.getValue() != null) {
ShardId shardId = ShardUtils.extractShardId(removalNotification.getKey().reader());
if (shardId != null) {
IndexShard shard = indexService.shard(shardId.id());
if (shard != null) {
shard.filterCache().onRemoval(removalNotification.getKey(), removalNotification.wasEvicted(), sizeInBytes(removalNotification.getValue()));
}
}
}
}
void onCached(FilterCacheKey cacheKey, DocIdSet cacheValue) {
ShardId shardId = ShardUtils.extractShardId(cacheKey.reader());
if (shardId != null) {
IndexShard shard = indexService.shard(shardId.id());
if (shard != null) {
shard.filterCache().onCached(cacheKey, sizeInBytes(cacheValue));
}
}
}
static long sizeInBytes(DocIdSet set) {
if (set instanceof FixedBitSet) {
return ((FixedBitSet) set).getBits().length * 8 + 16;
}
// only for empty ones
return 1;
}
public static class FilterCacheKey {
private final RemovalListener<WeightedFilterCache.FilterCacheKey, DocIdSet> removalListener;
private final IndexReader reader;
private final Object readerKey;
private final Object filterKey;
public FilterCacheKey(RemovalListener<WeightedFilterCache.FilterCacheKey, DocIdSet> removalListener, IndexReader reader, Object filterKey) {
this.removalListener = removalListener;
this.reader = reader;
// if we know, we will try and set the removal listener (for statistics)
// its ok that its not volatile because we make sure we only set it when the object is created before its shared between threads
@Nullable
public RemovalListener<WeightedFilterCache.FilterCacheKey, DocIdSet> removalListener;
public FilterCacheKey(Object readerKey, Object filterKey) {
this.readerKey = readerKey;
this.filterKey = filterKey;
}
public RemovalListener<WeightedFilterCache.FilterCacheKey, DocIdSet> removalListener() {
return removalListener;
}
public IndexReader reader() {
return this.reader;
}
public Object readerKey() {
return reader.getCoreCacheKey();
return readerKey;
}
public Object filterKey() {

5
src/main/java/org/elasticsearch/index/cache/id/IdReaderCache.java vendored
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.cache.id;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.common.bytes.HashedBytesArray;
/**
@ -27,10 +26,6 @@ import org.elasticsearch.common.bytes.HashedBytesArray;
*/
public interface IdReaderCache {
IndexReader reader();
Object readerCacheKey();
IdReaderTypeCache type(String type);
HashedBytesArray parentIdByDoc(String type, int docId);

17
src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java vendored
View File

@ -38,7 +38,6 @@ import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardUtils;
import org.elasticsearch.index.shard.service.IndexShard;
@ -201,6 +200,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
// now, build it back
for (Map.Entry<Object, Map<String, TypeBuilder>> entry : builders.entrySet()) {
Object readerKey = entry.getKey();
MapBuilder<String, SimpleIdReaderTypeCache> types = MapBuilder.newMapBuilder();
for (Map.Entry<String, TypeBuilder> typeBuilderEntry : entry.getValue().entrySet()) {
types.put(typeBuilderEntry.getKey(), new SimpleIdReaderTypeCache(typeBuilderEntry.getKey(),
@ -209,8 +209,9 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
typeBuilderEntry.getValue().parentIdsValues.toArray(new HashedBytesArray[typeBuilderEntry.getValue().parentIdsValues.size()]),
typeBuilderEntry.getValue().parentIdsOrdinals));
}
SimpleIdReaderCache readerCache = new SimpleIdReaderCache(cacheToReader.get(entry.getKey()), types.immutableMap());
idReaders.put(readerCache.readerCacheKey(), readerCache);
IndexReader indexReader = cacheToReader.get(readerKey);
SimpleIdReaderCache readerCache = new SimpleIdReaderCache(types.immutableMap(), ShardUtils.extractShardId(indexReader));
idReaders.put(readerKey, readerCache);
onCached(readerCache);
}
}
@ -218,9 +219,8 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
}
void onCached(SimpleIdReaderCache readerCache) {
ShardId shardId = ShardUtils.extractShardId(readerCache.reader());
if (shardId != null) {
IndexShard shard = indexService.shard(shardId.id());
if (readerCache.shardId != null) {
IndexShard shard = indexService.shard(readerCache.shardId.id());
if (shard != null) {
shard.idCache().onCached(readerCache.sizeInBytes());
}
@ -228,9 +228,8 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
}
void onRemoval(SimpleIdReaderCache readerCache) {
ShardId shardId = ShardUtils.extractShardId(readerCache.reader());
if (shardId != null) {
IndexShard shard = indexService.shard(shardId.id());
if (readerCache.shardId != null) {
IndexShard shard = indexService.shard(readerCache.shardId.id());
if (shard != null) {
shard.idCache().onCached(readerCache.sizeInBytes());
}

21
src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderCache.java vendored
View File

@ -20,32 +20,25 @@
package org.elasticsearch.index.cache.id.simple;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.index.cache.id.IdReaderCache;
import org.elasticsearch.index.cache.id.IdReaderTypeCache;
import org.elasticsearch.index.shard.ShardId;
/**
*
*/
public class SimpleIdReaderCache implements IdReaderCache {
private final IndexReader reader;
private final ImmutableMap<String, SimpleIdReaderTypeCache> types;
public SimpleIdReaderCache(IndexReader reader, ImmutableMap<String, SimpleIdReaderTypeCache> types) {
this.reader = reader;
@Nullable
public final ShardId shardId;
public SimpleIdReaderCache(ImmutableMap<String, SimpleIdReaderTypeCache> types, @Nullable ShardId shardId) {
this.types = types;
}
@Override
public IndexReader reader() {
return this.reader;
}
@Override
public Object readerCacheKey() {
return this.reader.getCoreCacheKey();
this.shardId = shardId;
}
@Override

6
src/main/java/org/elasticsearch/indices/cache/filter/IndicesFilterCache.java vendored
View File

@ -147,11 +147,13 @@ public class IndicesFilterCache extends AbstractComponent implements RemovalList
if (key == null) {
return;
}
key.removalListener().onRemoval(removalNotification);
if (key.removalListener != null) {
key.removalListener.onRemoval(removalNotification);
}
}
/**
* The reason we need this class ie because we need to clean all the filters that are associated
* The reason we need this class is because we need to clean all the filters that are associated
* with a reader. We don't want to do it every time a reader closes, since iterating over all the map
* is expensive. There doesn't seem to be a nicer way to do it (and maintaining a list per reader
* of the filters will cost more).