honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add field data stats

This commit is contained in:
Shay Banon 2013-01-24 11:38:18 +01:00
parent 613b746299
commit e8c1180ede
7 changed files with 82 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java
View File

@ -55,6 +55,12 @@ public class FieldDataStats implements Streamable, ToXContent {
return new ByteSizeValue(memorySize);
}
public static FieldDataStats readFieldDataStats(StreamInput in) throws IOException {
FieldDataStats stats = new FieldDataStats();
stats.readFrom(in);
return stats;
}
@Override
public void readFrom(StreamInput in) throws IOException {
memorySize = in.readVLong();

43
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
View File

@ -21,10 +21,14 @@ package org.elasticsearch.index.fielddata;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SegmentReader;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.FieldMapper;
import java.util.concurrent.Callable;
@ -41,14 +45,35 @@ public interface IndexFieldDataCache {
void clear(Index index, IndexReader reader);
interface Listener {
void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);
void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, @Nullable AtomicFieldData fieldData);
}
/**
* The resident field data cache is a *per field* cache that keeps all the values in memory.
*/
static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener {
static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener, RemovalListener<Object, AtomicFieldData> {
private final Index index;
private final FieldMapper.Names fieldNames;
private final FieldDataType fieldDataType;
private final Listener listener;
private final Cache<Object, AtomicFieldData> cache;
protected FieldBased(Cache<Object, AtomicFieldData> cache) {
this.cache = cache;
protected FieldBased(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener, CacheBuilder cache) {
this.index = index;
this.fieldNames = fieldNames;
this.fieldDataType = fieldDataType;
this.listener = listener;
cache.removalListener(this);
this.cache = cache.build();
}
@Override
public void onRemoval(RemovalNotification<Object, AtomicFieldData> notification) {
listener.onUnload(index, fieldNames, fieldDataType, notification.getValue());
}
@Override
@ -65,7 +90,9 @@ public interface IndexFieldDataCache {
if (context.reader() instanceof SegmentReader) {
((SegmentReader) context.reader()).addCoreClosedListener(FieldBased.this);
}
return indexFieldData.loadDirect(context);
AtomicFieldData fieldData = indexFieldData.loadDirect(context);
listener.onLoad(index, fieldNames, fieldDataType, fieldData);
return fieldData;
}
});
}
@ -88,15 +115,15 @@ public interface IndexFieldDataCache {
static class Resident extends FieldBased {
public Resident() {
super(CacheBuilder.newBuilder().<Object, AtomicFieldData>build());
public Resident(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder());
}
}
static class Soft extends FieldBased {
public Soft() {
super(CacheBuilder.newBuilder().softValues().<Object, AtomicFieldData>build());
public Soft(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder().softValues());
}
}
}

27
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
View File

@ -22,9 +22,11 @@ package org.elasticsearch.index.fielddata;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@ -38,7 +40,7 @@ import java.util.concurrent.ConcurrentMap;
/**
*/
public class IndexFieldDataService extends AbstractIndexComponent {
public class IndexFieldDataService extends AbstractIndexComponent implements IndexFieldDataCache.Listener {
private final static ImmutableMap<String, IndexFieldData.Builder> buildersByType;
private final static ImmutableMap<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
@ -70,6 +72,8 @@ public class IndexFieldDataService extends AbstractIndexComponent {
private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
private final CounterMetric memoryUsedInBytes = new CounterMetric();
public IndexFieldDataService(Index index) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
}
@ -103,9 +107,22 @@ public class IndexFieldDataService extends AbstractIndexComponent {
}
}
@Override
public void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData) {
assert index.equals(this.index);
memoryUsedInBytes.inc(fieldData.getMemorySizeInBytes());
}
@Override
public void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, @Nullable AtomicFieldData fieldData) {
assert index.equals(this.index);
if (fieldData != null) {
memoryUsedInBytes.dec(fieldData.getMemorySizeInBytes());
}
}
public FieldDataStats stats() {
// TODO: compute the memory size here...
return new FieldDataStats();
return new FieldDataStats(memoryUsedInBytes.count());
}
public <IFD extends IndexFieldData> IFD getForField(FieldMapper mapper) {
@ -136,9 +153,9 @@ public class IndexFieldDataService extends AbstractIndexComponent {
IndexFieldDataCache cache;
String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "resident"));
if ("resident".equals(cacheType)) {
cache = new IndexFieldDataCache.Resident();
cache = new IndexFieldDataCache.Resident(index, fieldNames, type, this);
} else if ("soft".equals(cacheType)) {
cache = new IndexFieldDataCache.Soft();
cache = new IndexFieldDataCache.Soft(index, fieldNames, type, this);
} else {
throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
}

5
src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View File

@ -42,6 +42,7 @@ import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.codec.CodecModule;
import org.elasticsearch.index.engine.IndexEngine;
import org.elasticsearch.index.engine.IndexEngineModule;
import org.elasticsearch.index.fielddata.FieldDataStats;
import org.elasticsearch.index.fielddata.IndexFieldDataModule;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.flush.FlushStats;
@ -186,6 +187,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
GetStats getStats = new GetStats();
SearchStats searchStats = new SearchStats();
CacheStats cacheStats = new CacheStats();
FieldDataStats fieldDataStats = new FieldDataStats();
MergeStats mergeStats = new MergeStats();
RefreshStats refreshStats = new RefreshStats();
FlushStats flushStats = new FlushStats();
@ -211,8 +213,9 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
flushStats.add(indexShard.flushStats());
}
cacheStats.add(indexService.cache().stats());
fieldDataStats.add(indexService.fieldData().stats());
}
return new NodeIndicesStats(storeStats, docsStats, indexingStats, getStats, searchStats, cacheStats, mergeStats, refreshStats, flushStats);
return new NodeIndicesStats(storeStats, docsStats, indexingStats, getStats, searchStats, cacheStats, fieldDataStats, mergeStats, refreshStats, flushStats);
}
/**

53
src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.index.cache.CacheStats;
import org.elasticsearch.index.fielddata.FieldDataStats;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.indexing.IndexingStats;
@ -40,8 +41,6 @@ import java.io.Serializable;
/**
* Global information on indices stats running on a specific node.
*
*
*/
public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
@ -57,6 +56,8 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
private CacheStats cacheStats;
private FieldDataStats fieldDataStats;
private MergeStats mergeStats;
private RefreshStats refreshStats;
@ -66,22 +67,19 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
NodeIndicesStats() {
}
public NodeIndicesStats(StoreStats storeStats, DocsStats docsStats, IndexingStats indexingStats, GetStats getStats, SearchStats searchStats, CacheStats cacheStats, MergeStats mergeStats, RefreshStats refreshStats, FlushStats flushStats) {
public NodeIndicesStats(StoreStats storeStats, DocsStats docsStats, IndexingStats indexingStats, GetStats getStats, SearchStats searchStats, CacheStats cacheStats, FieldDataStats fieldDataStats, MergeStats mergeStats, RefreshStats refreshStats, FlushStats flushStats) {
this.storeStats = storeStats;
this.docsStats = docsStats;
this.indexingStats = indexingStats;
this.getStats = getStats;
this.searchStats = searchStats;
this.cacheStats = cacheStats;
this.fieldDataStats = fieldDataStats;
this.mergeStats = mergeStats;
this.refreshStats = refreshStats;
this.flushStats = flushStats;
}
public StoreStats store() {
return this.storeStats;
}
/**
* The size of the index storage taken on the node.
*/
@ -89,64 +87,32 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
return storeStats;
}
public DocsStats docs() {
return this.docsStats;
}
public DocsStats getDocs() {
return this.docsStats;
}
public IndexingStats indexing() {
return indexingStats;
}
public IndexingStats getIndexing() {
return indexing();
}
public GetStats get() {
return this.getStats;
return indexingStats;
}
public GetStats getGet() {
return this.getStats;
}
public SearchStats search() {
return this.searchStats;
}
public SearchStats getSearch() {
return this.searchStats;
}
public CacheStats cache() {
return this.cacheStats;
}
public CacheStats getCache() {
return this.cache();
}
public MergeStats merge() {
return this.mergeStats;
return this.cacheStats;
}
public MergeStats getMerge() {
return this.mergeStats;
}
public RefreshStats refresh() {
return this.refreshStats;
}
public RefreshStats getRefresh() {
return this.refresh();
}
public FlushStats flush() {
return this.flushStats;
return refreshStats;
}
public FlushStats getFlush() {
@ -167,6 +133,7 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
getStats = GetStats.readGetStats(in);
searchStats = SearchStats.readSearchStats(in);
cacheStats = CacheStats.readCacheStats(in);
fieldDataStats = FieldDataStats.readFieldDataStats(in);
mergeStats = MergeStats.readMergeStats(in);
refreshStats = RefreshStats.readRefreshStats(in);
flushStats = FlushStats.readFlushStats(in);
@ -180,6 +147,7 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
getStats.writeTo(out);
searchStats.writeTo(out);
cacheStats.writeTo(out);
fieldDataStats.writeTo(out);
mergeStats.writeTo(out);
refreshStats.writeTo(out);
flushStats.writeTo(out);
@ -195,6 +163,7 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
getStats.toXContent(builder, params);
searchStats.toXContent(builder, params);
cacheStats.toXContent(builder, params);
fieldDataStats.toXContent(builder, params);
mergeStats.toXContent(builder, params);
refreshStats.toXContent(builder, params);
flushStats.toXContent(builder, params);

4
src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java
View File

@ -288,7 +288,7 @@ public class ChildSearchBenchmark {
statsResponse = client.admin().cluster().prepareNodesStats()
.setJvm(true).setIndices(true).execute().actionGet();
System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().cache().getIdCacheSize());
System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().getCache().getIdCacheSize());
System.out.println("--> Used heap size: " + statsResponse.nodes()[0].getJvm().getMem().getHeapUsed());
System.out.println("--> Running has_child query with score type");
@ -354,7 +354,7 @@ public class ChildSearchBenchmark {
statsResponse = client.admin().cluster().prepareNodesStats()
.setJvm(true).setIndices(true).execute().actionGet();
System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().cache().getIdCacheSize());
System.out.println("--> Id cache size: " + statsResponse.nodes()[0].getIndices().getCache().getIdCacheSize());
System.out.println("--> Used heap size: " + statsResponse.nodes()[0].getJvm().getMem().getHeapUsed());
client.close();

4
src/test/java/org/elasticsearch/test/integration/search/stats/SearchStatsTests.java
View File

@ -87,7 +87,7 @@ public class SearchStatsTests extends AbstractNodesTests {
assertThat(indicesStats.total().search().groupStats().get("group1").fetchTimeInMillis(), greaterThan(0l));
NodesStatsResponse nodeStats = client.admin().cluster().prepareNodesStats().execute().actionGet();
assertThat(nodeStats.nodes()[0].indices().search().total().queryCount(), greaterThan(0l));
assertThat(nodeStats.nodes()[0].indices().search().total().queryTimeInMillis(), greaterThan(0l));
assertThat(nodeStats.nodes()[0].indices().getSearch().total().queryCount(), greaterThan(0l));
assertThat(nodeStats.nodes()[0].indices().getSearch().total().queryTimeInMillis(), greaterThan(0l));
}
}