honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FieldData Stats: Add field data stats to indices stats API 

closes #2870
This commit is contained in:
Shay Banon 2013-04-07 18:30:24 -07:00
parent 86c1714bf3
commit 15d7ae5983
17 changed files with 392 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.cache.filter.FilterCacheStats;
import org.elasticsearch.index.cache.id.IdCacheStats;
import org.elasticsearch.index.fielddata.FieldDataStats;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.indexing.IndexingStats;
@ -76,6 +77,9 @@ public class CommonStats implements Streamable, ToXContent {
@Nullable
IdCacheStats idCache;
@Nullable
FieldDataStats fieldData;
public void add(CommonStats stats) {
if (docs == null) {
if (stats.getDocs() != null) {
@ -166,6 +170,15 @@ public class CommonStats implements Streamable, ToXContent {
} else {
idCache.add(stats.getIdCache());
}
if (fieldData == null) {
if (stats.getFieldData() != null) {
fieldData = new FieldDataStats();
fieldData.add(stats.getFieldData());
}
} else {
fieldData.add(stats.getFieldData());
}
}
@Nullable
@ -223,6 +236,11 @@ public class CommonStats implements Streamable, ToXContent {
return this.idCache;
}
@Nullable
public FieldDataStats getFieldData() {
return this.fieldData;
}
public static CommonStats readCommonStats(StreamInput in) throws IOException {
CommonStats stats = new CommonStats();
stats.readFrom(in);
@ -264,6 +282,9 @@ public class CommonStats implements Streamable, ToXContent {
if (in.readBoolean()) {
idCache = IdCacheStats.readIdCacheStats(in);
}
if (in.readBoolean()) {
fieldData = FieldDataStats.readFieldDataStats(in);
}
}
@Override
@ -328,13 +349,18 @@ public class CommonStats implements Streamable, ToXContent {
out.writeBoolean(true);
filterCache.writeTo(out);
}
if (idCache == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
idCache.writeTo(out);
}
if (fieldData == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
fieldData.writeTo(out);
}
}
// note, requires a wrapping object
@ -373,6 +399,9 @@ public class CommonStats implements Streamable, ToXContent {
if (idCache != null) {
idCache.toXContent(builder, params);
}
if (fieldData != null) {
fieldData.toXContent(builder, params);
}
return builder;
}
}

14
src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
View File

@ -47,6 +47,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest<IndicesStatsR
private boolean warmer = false;
private boolean filterCache = false;
private boolean idCache = false;
private boolean fieldData = false;
private String[] types = null;
private String[] groups = null;
@ -65,6 +66,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest<IndicesStatsR
warmer = true;
filterCache = true;
idCache = true;
fieldData = true;
types = null;
groups = null;
return this;
@ -85,6 +87,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest<IndicesStatsR
warmer = false;
filterCache = false;
idCache = false;
fieldData = false;
types = null;
groups = null;
return this;
@ -219,6 +222,15 @@ public class IndicesStatsRequest extends BroadcastOperationRequest<IndicesStatsR
return this.idCache;
}
public IndicesStatsRequest fieldData(boolean fieldData) {
this.fieldData = fieldData;
return this;
}
public boolean fieldData() {
return this.fieldData;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
@ -233,6 +245,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest<IndicesStatsR
out.writeBoolean(warmer);
out.writeBoolean(filterCache);
out.writeBoolean(idCache);
out.writeBoolean(fieldData);
if (types == null) {
out.writeVInt(0);
} else {
@ -265,6 +278,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest<IndicesStatsR
warmer = in.readBoolean();
filterCache = in.readBoolean();
idCache = in.readBoolean();
fieldData = in.readBoolean();
int size = in.readVInt();
if (size > 0) {
types = new String[size];

5
src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
View File

@ -124,6 +124,11 @@ public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder
return this;
}
public IndicesStatsRequestBuilder setFieldData(boolean fieldData) {
request.fieldData(fieldData);
return this;
}
@Override
protected void doExecute(ActionListener<IndicesStatsResponse> listener) {
((IndicesAdminClient) client).stats(request, listener);

3
src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
View File

@ -178,6 +178,9 @@ public class TransportIndicesStatsAction extends TransportBroadcastOperationActi
if (request.request.idCache()) {
stats.stats.idCache = indexShard.idCacheStats();
}
if (request.request.fieldData()) {
stats.stats.fieldData = indexShard.fieldDataStats();
}
return stats;
}

4
src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
View File

@ -29,11 +29,11 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
@Override
public void clear() {
cache.clear(index, fieldNames.indexName());
cache.clear(fieldNames.indexName());
}
@Override
public void clear(IndexReader reader) {
cache.clear(index, reader);
cache.clear(reader);
}
}

113
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
View File

@ -27,103 +27,154 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SegmentReader;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardUtils;
import org.elasticsearch.index.shard.service.IndexShard;
import java.util.concurrent.Callable;
/**
* A simple field data cache abstraction.
* A simple field data cache abstraction on the *index* level.
*/
public interface IndexFieldDataCache {
<FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(AtomicReaderContext context, IFD indexFieldData) throws Exception;
void clear(Index index);
/**
* Clears all the field data stored cached in on this index.
*/
void clear();
void clear(Index index, String fieldName);
/**
* Clears all the field data stored cached in on this index for the specified field name.
*/
void clear(String fieldName);
void clear(Index index, IndexReader reader);
void clear(IndexReader reader);
interface Listener {
void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);
void onLoad(FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);
void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, @Nullable AtomicFieldData fieldData);
void onUnload(FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes, @Nullable AtomicFieldData fieldData);
}
/**
* The resident field data cache is a *per field* cache that keeps all the values in memory.
*/
static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener, RemovalListener<Object, AtomicFieldData> {
private final Index index;
static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener, RemovalListener<FieldBased.Key, AtomicFieldData> {
@Nullable
private final IndexService indexService;
private final FieldMapper.Names fieldNames;
private final FieldDataType fieldDataType;
private final Listener listener;
private final Cache<Object, AtomicFieldData> cache;
private final Cache<Key, AtomicFieldData> cache;
protected FieldBased(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener, CacheBuilder cache) {
this.index = index;
protected FieldBased(@Nullable IndexService indexService, FieldMapper.Names fieldNames, FieldDataType fieldDataType, CacheBuilder cache) {
this.indexService = indexService;
this.fieldNames = fieldNames;
this.fieldDataType = fieldDataType;
this.listener = listener;
cache.removalListener(this);
this.cache = cache.build();
}
@Override
public void onRemoval(RemovalNotification<Object, AtomicFieldData> notification) {
listener.onUnload(index, fieldNames, fieldDataType, notification.wasEvicted(), notification.getValue());
}
@Override
public void onClose(SegmentReader owner) {
cache.invalidate(owner.getCoreCacheKey());
public void onRemoval(RemovalNotification<Key, AtomicFieldData> notification) {
if (notification.getKey() != null) {
notification.getKey().listener.onUnload(fieldNames, fieldDataType, notification.wasEvicted(), notification.getKey().sizeInBytes, notification.getValue());
}
}
@Override
public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
final Key key = new Key(context.reader().getCoreCacheKey());
//noinspection unchecked
return (FD) cache.get(context.reader().getCoreCacheKey(), new Callable<AtomicFieldData>() {
return (FD) cache.get(key, new Callable<AtomicFieldData>() {
@Override
public AtomicFieldData call() throws Exception {
if (context.reader() instanceof SegmentReader) {
((SegmentReader) context.reader()).addCoreClosedListener(FieldBased.this);
}
AtomicFieldData fieldData = indexFieldData.loadDirect(context);
listener.onLoad(index, fieldNames, fieldDataType, fieldData);
key.sizeInBytes = fieldData.getMemorySizeInBytes();
if (indexService != null) {
ShardId shardId = ShardUtils.extractShardId(context.reader());
if (shardId != null) {
IndexShard shard = indexService.shard(shardId.id());
if (shard != null) {
key.listener = shard.fieldData();
}
}
}
if (key.listener != null) {
key.listener.onLoad(fieldNames, fieldDataType, fieldData);
}
return fieldData;
}
});
}
@Override
public void clear(Index index) {
public void clear() {
cache.invalidateAll();
}
@Override
public void clear(Index index, String fieldName) {
public void clear(String fieldName) {
cache.invalidateAll();
}
@Override
public void clear(Index index, IndexReader reader) {
cache.invalidate(reader.getCoreCacheKey());
public void clear(IndexReader reader) {
cache.invalidate(new Key(reader.getCoreCacheKey()));
}
@Override
public void onClose(SegmentReader owner) {
cache.invalidate(new Key(owner.getCoreCacheKey()));
}
static class Key {
final Object readerKey;
@Nullable
Listener listener; // optional stats listener
long sizeInBytes = -1; // optional size in bytes (we keep it here in case the values are soft references)
Key(Object readerKey) {
this.readerKey = readerKey;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
Key key = (Key) o;
if (!readerKey.equals(key.readerKey)) return false;
return true;
}
@Override
public int hashCode() {
return readerKey.hashCode();
}
}
}
static class Resident extends FieldBased {
public Resident(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder());
public Resident(@Nullable IndexService indexService, FieldMapper.Names fieldNames, FieldDataType fieldDataType) {
super(indexService, fieldNames, fieldDataType, CacheBuilder.newBuilder());
}
}
static class Soft extends FieldBased {
public Soft(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder().softValues());
public Soft(@Nullable IndexService indexService, FieldMapper.Names fieldNames, FieldDataType fieldDataType) {
super(indexService, fieldNames, fieldDataType, CacheBuilder.newBuilder().softValues());
}
}
}

41
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
View File

@ -22,11 +22,9 @@ package org.elasticsearch.index.fielddata;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@ -34,6 +32,7 @@ import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.plain.*;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
@ -41,7 +40,7 @@ import java.util.concurrent.ConcurrentMap;
/**
*/
public class IndexFieldDataService extends AbstractIndexComponent implements IndexFieldDataCache.Listener {
public class IndexFieldDataService extends AbstractIndexComponent {
private final static ImmutableMap<String, IndexFieldData.Builder> buildersByType;
private final static ImmutableMap<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
@ -75,8 +74,7 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
private final IndicesFieldDataCache indicesFieldDataCache;
private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
private final CounterMetric memoryUsedInBytes = new CounterMetric();
private final CounterMetric evictions = new CounterMetric();
IndexService indexService;
public IndexFieldDataService(Index index) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, new IndicesFieldDataCache(ImmutableSettings.Builder.EMPTY_SETTINGS));
@ -88,6 +86,11 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
this.indicesFieldDataCache = indicesFieldDataCache;
}
// we need to "inject" the index service to not create cyclic dep
public void setIndexService(IndexService indexService) {
this.indexService = indexService;
}
public void clear() {
synchronized (loadedFieldData) {
for (IndexFieldData fieldData : loadedFieldData.values()) {
@ -112,28 +115,6 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
}
}
@Override
public void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData) {
assert index.equals(this.index);
memoryUsedInBytes.inc(fieldData.getMemorySizeInBytes());
}
@Override
public void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, @Nullable AtomicFieldData fieldData) {
assert index.equals(this.index);
if (fieldData != null) {
fieldData.close();
memoryUsedInBytes.dec(fieldData.getMemorySizeInBytes());
}
if (wasEvicted) {
evictions.inc();
}
}
public FieldDataStats stats() {
return new FieldDataStats(memoryUsedInBytes.count(), evictions.count());
}
public <IFD extends IndexFieldData> IFD getForField(FieldMapper mapper) {
return getForField(mapper.names(), mapper.fieldDataType());
}
@ -164,11 +145,11 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
// this means changing the node level settings is simple, just set the bounds there
String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "node"));
if ("resident".equals(cacheType)) {
cache = new IndexFieldDataCache.Resident(index, fieldNames, type, this);
cache = new IndexFieldDataCache.Resident(indexService, fieldNames, type);
} else if ("soft".equals(cacheType)) {
cache = new IndexFieldDataCache.Soft(index, fieldNames, type, this);
cache = new IndexFieldDataCache.Soft(indexService, fieldNames, type);
} else if ("node".equals(cacheType)) {
cache = indicesFieldDataCache.buildIndexFieldDataCache(index, fieldNames, type, this);
cache = indicesFieldDataCache.buildIndexFieldDataCache(indexService, index, fieldNames, type);
} else {
throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
}

64
src/main/java/org/elasticsearch/index/fielddata/ShardFieldData.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
/**
*/
public class ShardFieldData extends AbstractIndexShardComponent implements IndexFieldDataCache.Listener {
final CounterMetric evictionsMetric = new CounterMetric();
final CounterMetric totalMetric = new CounterMetric();
@Inject
public ShardFieldData(ShardId shardId, @IndexSettings Settings indexSettings) {
super(shardId, indexSettings);
}
public FieldDataStats stats() {
return new FieldDataStats(totalMetric.count(), evictionsMetric.count());
}
@Override
public void onLoad(FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData) {
totalMetric.inc(fieldData.getMemorySizeInBytes());
}
@Override
public void onUnload(FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes, @Nullable AtomicFieldData fieldData) {
if (wasEvicted) {
evictionsMetric.inc();
}
if (sizeInBytes != -1) {
totalMetric.dec(sizeInBytes);
} else if (fieldData != null) {
totalMetric.dec(fieldData.getMemorySizeInBytes());
}
}
}

32
src/main/java/org/elasticsearch/index/fielddata/ShardFieldDataModule.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.common.inject.AbstractModule;
/**
*/
public class ShardFieldDataModule extends AbstractModule {
@Override
protected void configure() {
bind(ShardFieldData.class).asEagerSingleton();
}
}

3
src/main/java/org/elasticsearch/index/service/InternalIndexService.java
View File

@ -40,6 +40,7 @@ import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineModule;
import org.elasticsearch.index.engine.IndexEngine;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.fielddata.ShardFieldDataModule;
import org.elasticsearch.index.gateway.IndexGateway;
import org.elasticsearch.index.gateway.IndexShardGatewayModule;
import org.elasticsearch.index.gateway.IndexShardGatewayService;
@ -157,6 +158,7 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
// inject workarounds for cyclic dep
indexCache.filter().setIndexService(this);
indexCache.idCache().setIndexService(this);
indexFieldData.setIndexService(this);
}
@Override
@ -325,6 +327,7 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
modules.add(new MergePolicyModule(indexSettings));
modules.add(new MergeSchedulerModule(indexSettings));
modules.add(new ShardFilterCacheModule());
modules.add(new ShardFieldDataModule());
modules.add(new ShardIdCacheModule());
modules.add(new TranslogModule(indexSettings));
modules.add(new EngineModule(indexSettings));

6
src/main/java/org/elasticsearch/index/shard/service/IndexShard.java
View File

@ -29,6 +29,8 @@ import org.elasticsearch.index.cache.id.IdCacheStats;
import org.elasticsearch.index.cache.id.ShardIdCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineException;
import org.elasticsearch.index.fielddata.FieldDataStats;
import org.elasticsearch.index.fielddata.ShardFieldData;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.get.ShardGetService;
@ -64,6 +66,8 @@ public interface IndexShard extends IndexShardComponent {
ShardIdCache idCache();
ShardFieldData fieldData();
ShardRouting routingEntry();
DocsStats docStats();
@ -88,6 +92,8 @@ public interface IndexShard extends IndexShardComponent {
IdCacheStats idCacheStats();
FieldDataStats fieldDataStats();
IndexShardState state();
Engine.Create prepareCreate(SourceToParse source) throws ElasticSearchException;

16
src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java
View File

@ -46,6 +46,8 @@ import org.elasticsearch.index.cache.filter.ShardFilterCache;
import org.elasticsearch.index.cache.id.IdCacheStats;
import org.elasticsearch.index.cache.id.ShardIdCache;
import org.elasticsearch.index.engine.*;
import org.elasticsearch.index.fielddata.FieldDataStats;
import org.elasticsearch.index.fielddata.ShardFieldData;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.get.ShardGetService;
@ -102,6 +104,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
private final ShardIndexWarmerService shardWarmerService;
private final ShardFilterCache shardFilterCache;
private final ShardIdCache shardIdCache;
private final ShardFieldData shardFieldData;
private final Object mutex = new Object();
private final String checkIndexOnStartup;
@ -125,7 +128,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
@Inject
public InternalIndexShard(ShardId shardId, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService, IndicesLifecycle indicesLifecycle, Store store, Engine engine, MergeSchedulerProvider mergeScheduler, Translog translog,
ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, ShardIndexingService indexingService, ShardGetService getService, ShardSearchService searchService, ShardIndexWarmerService shardWarmerService,
ShardFilterCache shardFilterCache, ShardIdCache shardIdCache) {
ShardFilterCache shardFilterCache, ShardIdCache shardIdCache, ShardFieldData shardFieldData) {
super(shardId, indexSettings);
this.indicesLifecycle = (InternalIndicesLifecycle) indicesLifecycle;
this.indexSettingsService = indexSettingsService;
@ -144,6 +147,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
this.shardWarmerService = shardWarmerService;
this.shardFilterCache = shardFilterCache;
this.shardIdCache = shardIdCache;
this.shardFieldData = shardFieldData;
state = IndexShardState.CREATED;
this.refreshInterval = indexSettings.getAsTime("engine.robin.refresh_interval", indexSettings.getAsTime(INDEX_REFRESH_INTERVAL, engine.defaultRefreshInterval()));
@ -201,6 +205,11 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
return this.shardIdCache;
}
@Override
public ShardFieldData fieldData() {
return this.shardFieldData;
}
@Override
public ShardRouting routingEntry() {
return this.shardRouting;
@ -467,6 +476,11 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
return shardFilterCache.stats();
}
@Override
public FieldDataStats fieldDataStats() {
return shardFieldData.stats();
}
@Override
public IdCacheStats idCacheStats() {
return shardIdCache.stats();

2
src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View File

@ -214,8 +214,8 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
flushStats.add(indexShard.flushStats());
filterCacheStats.add(indexShard.filterCacheStats());
idCacheStats.add(indexShard.idCacheStats());
fieldDataStats.add(indexShard.fieldDataStats());
}
fieldDataStats.add(indexService.fieldData().stats());
}
return new NodeIndicesStats(storeStats, docsStats, indexingStats, getStats, searchStats, fieldDataStats, mergeStats, refreshStats, flushStats, filterCacheStats, idCacheStats);
}

4
src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
View File

@ -109,6 +109,10 @@ public class NodeIndicesStats implements Streamable, Serializable, ToXContent {
return this.flushStats;
}
public FieldDataStats getFieldData() {
return fieldDataStats;
}
public FilterCacheStats getFilterCache() {
return this.filterCacheStats;
}

53
src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java vendored
View File

@ -23,6 +23,7 @@ import com.google.common.cache.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SegmentReader;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@ -34,6 +35,10 @@ import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardUtils;
import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.monitor.jvm.JvmInfo;
import java.util.concurrent.Callable;
@ -89,15 +94,15 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
cache.invalidateAll();
}
public IndexFieldDataCache buildIndexFieldDataCache(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache.Listener listener) {
return new IndexFieldCache(index, fieldNames, fieldDataType, listener);
public IndexFieldDataCache buildIndexFieldDataCache(@Nullable IndexService indexService, Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType) {
return new IndexFieldCache(indexService, index, fieldNames, fieldDataType);
}
@Override
public void onRemoval(RemovalNotification<Key, AtomicFieldData> notification) {
if (notification.getKey() != null) {
IndexFieldCache indexFieldCache = notification.getKey().indexCache;
indexFieldCache.listener.onUnload(indexFieldCache.index, indexFieldCache.fieldNames, indexFieldCache.fieldDataType, notification.wasEvicted(), notification.getValue());
IndexFieldCache indexCache = notification.getKey().indexCache;
notification.getKey().listener.onUnload(indexCache.fieldNames, indexCache.fieldDataType, notification.wasEvicted(), notification.getKey().sizeInBytes, notification.getValue());
}
}
@ -115,21 +120,22 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
*/
class IndexFieldCache implements IndexFieldDataCache, SegmentReader.CoreClosedListener {
@Nullable
private final IndexService indexService;
final Index index;
final FieldMapper.Names fieldNames;
final FieldDataType fieldDataType;
final Listener listener;
IndexFieldCache(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
IndexFieldCache(@Nullable IndexService indexService, Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType) {
this.indexService = indexService;
this.index = index;
this.fieldNames = fieldNames;
this.fieldDataType = fieldDataType;
this.listener = listener;
}
@Override
public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
Key key = new Key(this, context.reader().getCoreCacheKey());
final Key key = new Key(this, context.reader().getCoreCacheKey());
//noinspection unchecked
return (FD) cache.get(key, new Callable<AtomicFieldData>() {
@Override
@ -138,7 +144,21 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
((SegmentReader) context.reader()).addCoreClosedListener(IndexFieldCache.this);
}
AtomicFieldData fieldData = indexFieldData.loadDirect(context);
listener.onLoad(index, indexFieldData.getFieldNames(), fieldDataType, fieldData);
if (indexService != null) {
ShardId shardId = ShardUtils.extractShardId(context.reader());
if (shardId != null) {
IndexShard shard = indexService.shard(shardId.id());
if (shard != null) {
key.listener = shard.fieldData();
}
}
}
if (key.listener != null) {
key.listener.onLoad(fieldNames, fieldDataType, fieldData);
}
return fieldData;
}
});
@ -150,7 +170,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
}
@Override
public void clear(Index index) {
public void clear() {
for (Key key : cache.asMap().keySet()) {
if (key.indexCache.index.equals(index)) {
cache.invalidate(key);
@ -159,7 +179,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
}
@Override
public void clear(Index index, String fieldName) {
public void clear(String fieldName) {
for (Key key : cache.asMap().keySet()) {
if (key.indexCache.index.equals(index)) {
if (key.indexCache.fieldNames.fullName().equals(fieldName)) {
@ -170,7 +190,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
}
@Override
public void clear(Index index, IndexReader reader) {
public void clear(IndexReader reader) {
cache.invalidate(new Key(this, reader.getCoreCacheKey()));
}
}
@ -179,6 +199,11 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
public final IndexFieldCache indexCache;
public final Object readerKey;
@Nullable
public IndexFieldDataCache.Listener listener; // optional stats listener
long sizeInBytes = -1; // optional size in bytes (we keep it here in case the values are soft references)
Key(IndexFieldCache indexCache, Object readerKey) {
this.indexCache = indexCache;
this.readerKey = readerKey;
@ -187,13 +212,9 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Key key = (Key) o;
if (!indexCache.equals(key.indexCache)) return false;
if (!readerKey.equals(key.readerKey)) return false;
return true;
}

42
src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
View File

@ -83,6 +83,9 @@ public class RestIndicesStatsAction extends BaseRestHandler {
controller.registerHandler(GET, "/_stats/id_cache", new RestIdCacheStatsHandler());
controller.registerHandler(GET, "/{index}/_stats/id_cache", new RestIdCacheStatsHandler());
controller.registerHandler(GET, "/_stats/fielddata", new RestFieldDataStatsHandler());
controller.registerHandler(GET, "/{index}/_stats/fielddata", new RestFieldDataStatsHandler());
}
@Override
@ -116,6 +119,7 @@ public class RestIndicesStatsAction extends BaseRestHandler {
indicesStatsRequest.warmer(request.paramAsBoolean("warmer", indicesStatsRequest.warmer()));
indicesStatsRequest.filterCache(request.paramAsBoolean("filter_cache", indicesStatsRequest.filterCache()));
indicesStatsRequest.idCache(request.paramAsBoolean("id_cache", indicesStatsRequest.idCache()));
indicesStatsRequest.idCache(request.paramAsBoolean("fielddata", indicesStatsRequest.fieldData()));
client.admin().indices().stats(indicesStatsRequest, new ActionListener<IndicesStatsResponse>() {
@Override
@ -535,6 +539,44 @@ public class RestIndicesStatsAction extends BaseRestHandler {
}
}
class RestFieldDataStatsHandler implements RestHandler {
@Override
public void handleRequest(final RestRequest request, final RestChannel channel) {
IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
indicesStatsRequest.listenerThreaded(false);
indicesStatsRequest.clear().fieldData(true);
indicesStatsRequest.indices(splitIndices(request.param("index")));
indicesStatsRequest.types(splitTypes(request.param("types")));
client.admin().indices().stats(indicesStatsRequest, new ActionListener<IndicesStatsResponse>() {
@Override
public void onResponse(IndicesStatsResponse response) {
try {
XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
builder.startObject();
builder.field("ok", true);
buildBroadcastShardsHeader(builder, response);
response.toXContent(builder, request);
builder.endObject();
channel.sendResponse(new XContentRestResponse(request, OK, builder));
} catch (Throwable e) {
onFailure(e);
}
}
@Override
public void onFailure(Throwable e) {
try {
channel.sendResponse(new XContentThrowableRestResponse(request, e));
} catch (IOException e1) {
logger.error("Failed to send failure response", e1);
}
}
});
}
}
class RestRefreshStatsHandler implements RestHandler {
@Override

43
src/test/java/org/elasticsearch/test/integration/indices/cache/ClearCacheTests.java → src/test/java/org/elasticsearch/test/integration/indices/cache/CacheTests.java vendored
View File

@ -20,10 +20,12 @@
package org.elasticsearch.test.integration.indices.cache;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@ -38,13 +40,13 @@ import static org.hamcrest.Matchers.greaterThan;
/**
*/
@Test
public class ClearCacheTests extends AbstractNodesTests {
public class CacheTests extends AbstractNodesTests {
private Client client;
@BeforeClass
public void createNodes() throws Exception {
startNode("node1", ImmutableSettings.settingsBuilder().put("index.cache.stats.refresh_interval", 0));
startNode("node1");
client = getClient();
}
@ -60,20 +62,57 @@ public class ClearCacheTests extends AbstractNodesTests {
@Test
public void testClearCacheFilterKeys() {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet();
client.prepareIndex("test", "type", "1").setSource("field", "value").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats = client.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats = client.admin().indices().prepareStats("test").clear().setFilterCache(true).execute().actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
SearchResponse searchResponse = client.prepareSearch().setQuery(filteredQuery(matchAllQuery(), FilterBuilders.termFilter("field", "value").cacheKey("test_key"))).execute().actionGet();
assertThat(searchResponse.getHits().getHits().length, equalTo(1));
nodesStats = client.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
indicesStats = client.admin().indices().prepareStats("test").clear().setFilterCache(true).execute().actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
client.admin().indices().prepareClearCache().setFilterKeys("test_key").execute().actionGet();
nodesStats = client.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
indicesStats = client.admin().indices().prepareStats("test").clear().setFilterCache(true).execute().actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
}
@Test
public void testFieldDataStats() {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", 1)).execute().actionGet();
client.prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
client.prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
NodesStatsResponse nodesStats = client.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
IndicesStatsResponse indicesStats = client.admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
// sort to load it to field data...
client.prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
client.prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
nodesStats = client.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
indicesStats = client.admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
client.admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
nodesStats = client.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
indicesStats = client.admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
}
}