add field data stats

src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java

@@ -55,6 +55,12 @@ public class FieldDataStats implements Streamable, ToXContent {
         return new ByteSizeValue(memorySize);
     }
 
+    public static FieldDataStats readFieldDataStats(StreamInput in) throws IOException {
+        FieldDataStats stats = new FieldDataStats();
+        stats.readFrom(in);
+        return stats;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         memorySize = in.readVLong();

src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java

@@ -21,10 +21,14 @@ package org.elasticsearch.index.fielddata;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.SegmentReader;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.FieldMapper;
 
 import java.util.concurrent.Callable;
 
@@ -41,14 +45,35 @@ public interface IndexFieldDataCache {
 
     void clear(Index index, IndexReader reader);
 
+    interface Listener {
+
+        void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);
+
+        void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, @Nullable AtomicFieldData fieldData);
+    }
+
     /**
      * The resident field data cache is a *per field* cache that keeps all the values in memory.
      */
-    static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener {
+    static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener, RemovalListener<Object, AtomicFieldData> {
+        private final Index index;
+        private final FieldMapper.Names fieldNames;
+        private final FieldDataType fieldDataType;
+        private final Listener listener;
         private final Cache<Object, AtomicFieldData> cache;
 
-        protected FieldBased(Cache<Object, AtomicFieldData> cache) {
+        protected FieldBased(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener, CacheBuilder cache) {
-            this.cache = cache;
+            this.index = index;
+            this.fieldNames = fieldNames;
+            this.fieldDataType = fieldDataType;
+            this.listener = listener;
+            cache.removalListener(this);
+            this.cache = cache.build();
+        }
+
+        @Override
+        public void onRemoval(RemovalNotification<Object, AtomicFieldData> notification) {
+            listener.onUnload(index, fieldNames, fieldDataType, notification.getValue());
         }
 
         @Override
@@ -65,7 +90,9 @@ public interface IndexFieldDataCache {
                     if (context.reader() instanceof SegmentReader) {
                         ((SegmentReader) context.reader()).addCoreClosedListener(FieldBased.this);
                     }
-                    return indexFieldData.loadDirect(context);
+                    AtomicFieldData fieldData = indexFieldData.loadDirect(context);
+                    listener.onLoad(index, fieldNames, fieldDataType, fieldData);
+                    return fieldData;
                 }
             });
         }
@@ -88,15 +115,15 @@ public interface IndexFieldDataCache {
 
     static class Resident extends FieldBased {
 
-        public Resident() {
+        public Resident(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
-            super(CacheBuilder.newBuilder().<Object, AtomicFieldData>build());
+            super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder());
         }
     }
 
     static class Soft extends FieldBased {
 
-        public Soft() {
+        public Soft(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
-            super(CacheBuilder.newBuilder().softValues().<Object, AtomicFieldData>build());
+            super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder().softValues());
         }
     }
 }

src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java

@@ -22,9 +22,11 @@ package org.elasticsearch.index.fielddata;
 import com.google.common.collect.ImmutableMap;
 import org.apache.lucene.index.IndexReader;
 import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -38,7 +40,7 @@ import java.util.concurrent.ConcurrentMap;
 
 /**
  */
-public class IndexFieldDataService extends AbstractIndexComponent {
+public class IndexFieldDataService extends AbstractIndexComponent implements IndexFieldDataCache.Listener {
 
     private final static ImmutableMap<String, IndexFieldData.Builder> buildersByType;
     private final static ImmutableMap<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
@@ -70,6 +72,8 @@ public class IndexFieldDataService extends AbstractIndexComponent {
 
     private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
 
+    private final CounterMetric memoryUsedInBytes = new CounterMetric();
+
     public IndexFieldDataService(Index index) {
         this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
     }
@@ -103,9 +107,22 @@ public class IndexFieldDataService extends AbstractIndexComponent {
         }
     }
 
+    @Override
+    public void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData) {
+        assert index.equals(this.index);
+        memoryUsedInBytes.inc(fieldData.getMemorySizeInBytes());
+    }
+
+    @Override
+    public void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, @Nullable AtomicFieldData fieldData) {
+        assert index.equals(this.index);
+        if (fieldData != null) {
+            memoryUsedInBytes.dec(fieldData.getMemorySizeInBytes());
+        }
+    }
+
     public FieldDataStats stats() {
-        // TODO: compute the memory size here...
+        return new FieldDataStats(memoryUsedInBytes.count());
-        return new FieldDataStats();
     }
 
     public <IFD extends IndexFieldData> IFD getForField(FieldMapper mapper) {
@@ -136,9 +153,9 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                     IndexFieldDataCache cache;
                     String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "resident"));
                     if ("resident".equals(cacheType)) {
-                        cache = new IndexFieldDataCache.Resident();
+                        cache = new IndexFieldDataCache.Resident(index, fieldNames, type, this);
                     } else if ("soft".equals(cacheType)) {
-                        cache = new IndexFieldDataCache.Soft();
+                        cache = new IndexFieldDataCache.Soft(index, fieldNames, type, this);
                     } else {
                         throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
                     }

src/main/java/org/elasticsearch/indices/InternalIndicesService.java

@@ -42,6 +42,7 @@ import org.elasticsearch.index.cache.IndexCacheModule;
 import org.elasticsearch.index.codec.CodecModule;
 import org.elasticsearch.index.engine.IndexEngine;
 import org.elasticsearch.index.engine.IndexEngineModule;
+import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.fielddata.IndexFieldDataModule;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.flush.FlushStats;
@@ -186,6 +187,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
         GetStats getStats = new GetStats();
         SearchStats searchStats = new SearchStats();
         CacheStats cacheStats = new CacheStats();
+        FieldDataStats fieldDataStats = new FieldDataStats();
         MergeStats mergeStats = new MergeStats();
         RefreshStats refreshStats = new RefreshStats();
         FlushStats flushStats = new FlushStats();
@@ -211,8 +213,9 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
                 flushStats.add(indexShard.flushStats());
             }
             cacheStats.add(indexService.cache().stats());
+            fieldDataStats.add(indexService.fieldData().stats());
         }
-        return new NodeIndicesStats(storeStats, docsStats, indexingStats, getStats, searchStats, cacheStats, mergeStats, refreshStats, flushStats);
+        return new NodeIndicesStats(storeStats, docsStats, indexingStats, getStats, searchStats, cacheStats, fieldDataStats, mergeStats, refreshStats, flushStats);
     }
 
     /**

src/main/java/org/elasticsearch/indices/NodeIndicesStats.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.index.cache.CacheStats;
+import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.indexing.IndexingStats;
@@ -40,8 +41,6 @@ import java.io.Serializable;
 
 /**
  * Global information on indices stats running on a specific node.
- *
- *
  */
 public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
 
@@ -57,6 +56,8 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
 
     private CacheStats cacheStats;
 
+    private FieldDataStats fieldDataStats;
+
     private MergeStats mergeStats;
 
     private RefreshStats refreshStats;
@@ -66,22 +67,19 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
     NodeIndicesStats() {
     }
 
-    public NodeIndicesStats(StoreStats storeStats, DocsStats docsStats, IndexingStats indexingStats, GetStats getStats, SearchStats searchStats, CacheStats cacheStats, MergeStats mergeStats, RefreshStats refreshStats, FlushStats flushStats) {
+    public NodeIndicesStats(StoreStats storeStats, DocsStats docsStats, IndexingStats indexingStats, GetStats getStats, SearchStats searchStats, CacheStats cacheStats, FieldDataStats fieldDataStats, MergeStats mergeStats, RefreshStats refreshStats, FlushStats flushStats) {
         this.storeStats = storeStats;
         this.docsStats = docsStats;
         this.indexingStats = indexingStats;
         this.getStats = getStats;
         this.searchStats = searchStats;
         this.cacheStats = cacheStats;
+        this.fieldDataStats = fieldDataStats;
         this.mergeStats = mergeStats;
         this.refreshStats = refreshStats;
         this.flushStats = flushStats;
     }
 
-    public StoreStats store() {
-        return this.storeStats;
-    }
-
     /**
      * The size of the index storage taken on the node.
      */
@@ -89,64 +87,32 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
         return storeStats;
     }
 
-    public DocsStats docs() {
-        return this.docsStats;
-    }
-
     public DocsStats getDocs() {
         return this.docsStats;
     }
 
-    public IndexingStats indexing() {
-        return indexingStats;
-    }
-
     public IndexingStats getIndexing() {
-        return indexing();
+        return indexingStats;
-    }
-
-    public GetStats get() {
-        return this.getStats;
     }
 
     public GetStats getGet() {
         return this.getStats;
     }
 
-    public SearchStats search() {
-        return this.searchStats;
-    }
-
     public SearchStats getSearch() {
         return this.searchStats;
     }
 
-    public CacheStats cache() {
-        return this.cacheStats;
-    }
-
     public CacheStats getCache() {
-        return this.cache();
+        return this.cacheStats;
-    }
-
-    public MergeStats merge() {
-        return this.mergeStats;
     }
 
     public MergeStats getMerge() {
         return this.mergeStats;
     }
 
-    public RefreshStats refresh() {
-        return this.refreshStats;
-    }
-
     public RefreshStats getRefresh() {
-        return this.refresh();
+        return refreshStats;
-    }
-
-    public FlushStats flush() {
-        return this.flushStats;
     }
 
     public FlushStats getFlush() {
@@ -167,6 +133,7 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
         getStats = GetStats.readGetStats(in);
         searchStats = SearchStats.readSearchStats(in);
         cacheStats = CacheStats.readCacheStats(in);
+        fieldDataStats = FieldDataStats.readFieldDataStats(in);
         mergeStats = MergeStats.readMergeStats(in);
         refreshStats = RefreshStats.readRefreshStats(in);
         flushStats = FlushStats.readFlushStats(in);
@@ -180,6 +147,7 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
         getStats.writeTo(out);
         searchStats.writeTo(out);
         cacheStats.writeTo(out);
+        fieldDataStats.writeTo(out);
         mergeStats.writeTo(out);
         refreshStats.writeTo(out);
         flushStats.writeTo(out);
@@ -195,6 +163,7 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
         getStats.toXContent(builder, params);
         searchStats.toXContent(builder, params);
         cacheStats.toXContent(builder, params);
+        fieldDataStats.toXContent(builder, params);
         mergeStats.toXContent(builder, params);
         refreshStats.toXContent(builder, params);
         flushStats.toXContent(builder, params);

src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java

@@ -288,7 +288,7 @@ public class ChildSearchBenchmark {
         statsResponse = client.admin().cluster().prepareNodesStats()
                 .setJvm(true).setIndices(true).execute().actionGet();
 
-        System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().cache().getIdCacheSize());
+        System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().getCache().getIdCacheSize());
         System.out.println("--> Used heap size: " + statsResponse.nodes()[0].getJvm().getMem().getHeapUsed());
 
         System.out.println("--> Running has_child query with score type");
@@ -354,7 +354,7 @@ public class ChildSearchBenchmark {
         statsResponse = client.admin().cluster().prepareNodesStats()
                 .setJvm(true).setIndices(true).execute().actionGet();
 
-        System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().cache().getIdCacheSize());
+        System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().getCache().getIdCacheSize());
         System.out.println("--> Used heap size: " + statsResponse.nodes()[0].getJvm().getMem().getHeapUsed());
 
         client.close();

src/test/java/org/elasticsearch/test/integration/search/stats/SearchStatsTests.java

@@ -87,7 +87,7 @@ public class SearchStatsTests extends AbstractNodesTests {
         assertThat(indicesStats.total().search().groupStats().get("group1").fetchTimeInMillis(), greaterThan(0l));
 
         NodesStatsResponse nodeStats = client.admin().cluster().prepareNodesStats().execute().actionGet();
-        assertThat(nodeStats.nodes()[0].indices().search().total().queryCount(), greaterThan(0l));
+        assertThat(nodeStats.nodes()[0].indices().getSearch().total().queryCount(), greaterThan(0l));
-        assertThat(nodeStats.nodes()[0].indices().search().total().queryTimeInMillis(), greaterThan(0l));
+        assertThat(nodeStats.nodes()[0].indices().getSearch().total().queryTimeInMillis(), greaterThan(0l));
     }
 }