diff --git a/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java b/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java index 2ef0b3ce7b0..0195758d21a 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java +++ b/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java @@ -55,6 +55,12 @@ public class FieldDataStats implements Streamable, ToXContent { return new ByteSizeValue(memorySize); } + public static FieldDataStats readFieldDataStats(StreamInput in) throws IOException { + FieldDataStats stats = new FieldDataStats(); + stats.readFrom(in); + return stats; + } + @Override public void readFrom(StreamInput in) throws IOException { memorySize = in.readVLong(); diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java index 014943f46a6..15c6b1a1505 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java +++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java @@ -21,10 +21,14 @@ package org.elasticsearch.index.fielddata; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SegmentReader; +import org.elasticsearch.common.Nullable; import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.FieldMapper; import java.util.concurrent.Callable; @@ -41,14 +45,35 @@ public interface IndexFieldDataCache { void clear(Index index, IndexReader reader); + interface Listener { + + void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData); + + void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, @Nullable AtomicFieldData fieldData); + } + /** * The resident field data cache is a *per field* cache that keeps all the values in memory. */ - static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener { + static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener, RemovalListener { + private final Index index; + private final FieldMapper.Names fieldNames; + private final FieldDataType fieldDataType; + private final Listener listener; private final Cache cache; - protected FieldBased(Cache cache) { - this.cache = cache; + protected FieldBased(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener, CacheBuilder cache) { + this.index = index; + this.fieldNames = fieldNames; + this.fieldDataType = fieldDataType; + this.listener = listener; + cache.removalListener(this); + this.cache = cache.build(); + } + + @Override + public void onRemoval(RemovalNotification notification) { + listener.onUnload(index, fieldNames, fieldDataType, notification.getValue()); } @Override @@ -65,7 +90,9 @@ public interface IndexFieldDataCache { if (context.reader() instanceof SegmentReader) { ((SegmentReader) context.reader()).addCoreClosedListener(FieldBased.this); } - return indexFieldData.loadDirect(context); + AtomicFieldData fieldData = indexFieldData.loadDirect(context); + listener.onLoad(index, fieldNames, fieldDataType, fieldData); + return fieldData; } }); } @@ -88,15 +115,15 @@ public interface IndexFieldDataCache { static class Resident extends FieldBased { - public Resident() { - super(CacheBuilder.newBuilder().build()); + public Resident(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) { + super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder()); } } static class Soft extends FieldBased { - public Soft() { - super(CacheBuilder.newBuilder().softValues().build()); + public Soft(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) { + super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder().softValues()); } } } diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 862e25cf86c..eddbf9f4925 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -22,9 +22,11 @@ package org.elasticsearch.index.fielddata; import com.google.common.collect.ImmutableMap; import org.apache.lucene.index.IndexReader; import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -38,7 +40,7 @@ import java.util.concurrent.ConcurrentMap; /** */ -public class IndexFieldDataService extends AbstractIndexComponent { +public class IndexFieldDataService extends AbstractIndexComponent implements IndexFieldDataCache.Listener { private final static ImmutableMap buildersByType; private final static ImmutableMap, IndexFieldData.Builder> buildersByTypeAndFormat; @@ -70,6 +72,8 @@ public class IndexFieldDataService extends AbstractIndexComponent { private final ConcurrentMap loadedFieldData = ConcurrentCollections.newConcurrentMap(); + private final CounterMetric memoryUsedInBytes = new CounterMetric(); + public IndexFieldDataService(Index index) { this(index, ImmutableSettings.Builder.EMPTY_SETTINGS); } @@ -103,9 +107,22 @@ public class IndexFieldDataService extends AbstractIndexComponent { } } + @Override + public void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData) { + assert index.equals(this.index); + memoryUsedInBytes.inc(fieldData.getMemorySizeInBytes()); + } + + @Override + public void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, @Nullable AtomicFieldData fieldData) { + assert index.equals(this.index); + if (fieldData != null) { + memoryUsedInBytes.dec(fieldData.getMemorySizeInBytes()); + } + } + public FieldDataStats stats() { - // TODO: compute the memory size here... - return new FieldDataStats(); + return new FieldDataStats(memoryUsedInBytes.count()); } public IFD getForField(FieldMapper mapper) { @@ -136,9 +153,9 @@ public class IndexFieldDataService extends AbstractIndexComponent { IndexFieldDataCache cache; String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "resident")); if ("resident".equals(cacheType)) { - cache = new IndexFieldDataCache.Resident(); + cache = new IndexFieldDataCache.Resident(index, fieldNames, type, this); } else if ("soft".equals(cacheType)) { - cache = new IndexFieldDataCache.Soft(); + cache = new IndexFieldDataCache.Soft(index, fieldNames, type, this); } else { throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]"); } diff --git a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java index 52e0a08c1f3..8c560bf07e8 100644 --- a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java +++ b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java @@ -42,6 +42,7 @@ import org.elasticsearch.index.cache.IndexCacheModule; import org.elasticsearch.index.codec.CodecModule; import org.elasticsearch.index.engine.IndexEngine; import org.elasticsearch.index.engine.IndexEngineModule; +import org.elasticsearch.index.fielddata.FieldDataStats; import org.elasticsearch.index.fielddata.IndexFieldDataModule; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.flush.FlushStats; @@ -186,6 +187,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent Id cache size: " + statsResponse.nodes()[0].getIndices().cache().getIdCacheSize()); + System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().getCache().getIdCacheSize()); System.out.println("--> Used heap size: " + statsResponse.nodes()[0].getJvm().getMem().getHeapUsed()); System.out.println("--> Running has_child query with score type"); @@ -354,7 +354,7 @@ public class ChildSearchBenchmark { statsResponse = client.admin().cluster().prepareNodesStats() .setJvm(true).setIndices(true).execute().actionGet(); - System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().cache().getIdCacheSize()); + System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().getCache().getIdCacheSize()); System.out.println("--> Used heap size: " + statsResponse.nodes()[0].getJvm().getMem().getHeapUsed()); client.close(); diff --git a/src/test/java/org/elasticsearch/test/integration/search/stats/SearchStatsTests.java b/src/test/java/org/elasticsearch/test/integration/search/stats/SearchStatsTests.java index da57dfc2de7..57e103101f6 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/stats/SearchStatsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/stats/SearchStatsTests.java @@ -87,7 +87,7 @@ public class SearchStatsTests extends AbstractNodesTests { assertThat(indicesStats.total().search().groupStats().get("group1").fetchTimeInMillis(), greaterThan(0l)); NodesStatsResponse nodeStats = client.admin().cluster().prepareNodesStats().execute().actionGet(); - assertThat(nodeStats.nodes()[0].indices().search().total().queryCount(), greaterThan(0l)); - assertThat(nodeStats.nodes()[0].indices().search().total().queryTimeInMillis(), greaterThan(0l)); + assertThat(nodeStats.nodes()[0].indices().getSearch().total().queryCount(), greaterThan(0l)); + assertThat(nodeStats.nodes()[0].indices().getSearch().total().queryTimeInMillis(), greaterThan(0l)); } }