From 5e4343b48b44fa3b77630e2676b4889e59dd8657 Mon Sep 17 00:00:00 2001 From: kimchy Date: Sun, 16 May 2010 03:29:56 +0300 Subject: [PATCH] change how filter cache works, do weak/soft on IndexReader key, so we release the IndexReader on memory constraints --- .../cache/filter/soft/SoftFilterCache.java | 11 +++++------ .../AbstractConcurrentMapFilterCache.java | 17 +++++++++++------ .../cache/filter/weak/WeakFilterCache.java | 11 +++++------ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java index 68867d375c8..b5a20dafd06 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.cache.filter.soft; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.elasticsearch.index.Index; @@ -32,19 +33,17 @@ import org.elasticsearch.util.settings.Settings; import java.util.concurrent.ConcurrentMap; /** - * @author kimchy (Shay Banon) + * A soft reference based filter cache that has soft keys on the IndexReader. + * + * @author kimchy (shay.banon) */ public class SoftFilterCache extends AbstractConcurrentMapFilterCache { @Inject public SoftFilterCache(Index index, @IndexSettings Settings indexSettings, ThreadPool threadPool) { - super(index, indexSettings, threadPool); + super(index, indexSettings, threadPool, new MapMaker().softKeys().>makeMap()); } @Override public String type() { return "soft"; } - - @Override protected ConcurrentMap buildMap() { - return new MapMaker().softValues().makeMap(); - } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java index 9b97b89fb70..a2d7d0e3cb8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java @@ -39,7 +39,9 @@ import static org.elasticsearch.util.concurrent.ConcurrentCollections.*; import static org.elasticsearch.util.lucene.docidset.DocIdSets.*; /** - * @author kimchy (Shay Banon) + * A base concurrent filter cache that accepts the actual cache to use. + * + * @author kimchy (shay.banon) */ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComponent implements FilterCache { @@ -49,14 +51,15 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp private final Future scheduleFuture; - protected AbstractConcurrentMapFilterCache(Index index, @IndexSettings Settings indexSettings, ThreadPool threadPool) { + protected AbstractConcurrentMapFilterCache(Index index, @IndexSettings Settings indexSettings, ThreadPool threadPool, + ConcurrentMap> cache) { super(index, indexSettings); + this.cache = cache; this.readerCleanerSchedule = componentSettings.getAsTime("reader_cleaner_schedule", TimeValue.timeValueMinutes(1)); - logger.debug("Using [" + type() + "] filter cache with reader_cleaner_schedule[{}]", readerCleanerSchedule); + logger.debug("Using [" + type() + "] filter cache with reader_cleaner_schedule [{}]", readerCleanerSchedule); - this.cache = newConcurrentMap(); this.scheduleFuture = threadPool.scheduleWithFixedDelay(new IndexReaderCleaner(), readerCleanerSchedule); } @@ -88,7 +91,9 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp } } - protected abstract ConcurrentMap buildMap(); + protected ConcurrentMap buildFilterMap() { + return newConcurrentMap(); + } private class FilterCacheFilterWrapper extends Filter { @@ -101,7 +106,7 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { ConcurrentMap cachedFilters = cache.get(reader); if (cachedFilters == null) { - cachedFilters = buildMap(); + cachedFilters = buildFilterMap(); cache.putIfAbsent(reader, cachedFilters); } DocIdSet docIdSet = cachedFilters.get(filter); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java index 5ba16b859bc..9352493411b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.cache.filter.weak; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.elasticsearch.index.Index; @@ -32,19 +33,17 @@ import org.elasticsearch.util.settings.Settings; import java.util.concurrent.ConcurrentMap; /** - * @author kimchy (Shay Banon) + * A weak reference based filter cache that has weak keys on the IndexReader. + * + * @author kimchy (shay.banon) */ public class WeakFilterCache extends AbstractConcurrentMapFilterCache { @Inject public WeakFilterCache(Index index, @IndexSettings Settings indexSettings, ThreadPool threadPool) { - super(index, indexSettings, threadPool); + super(index, indexSettings, threadPool, new MapMaker().weakKeys().>makeMap()); } @Override public String type() { return "weak"; } - - @Override protected ConcurrentMap buildMap() { - return new MapMaker().weakValues().makeMap(); - } } \ No newline at end of file