parent
e01879a698
commit
a7da27c714
|
@ -26,6 +26,7 @@ import com.google.common.cache.RemovalNotification;
|
||||||
import org.apache.lucene.index.AtomicReaderContext;
|
import org.apache.lucene.index.AtomicReaderContext;
|
||||||
import org.apache.lucene.index.IndexReader;
|
import org.apache.lucene.index.IndexReader;
|
||||||
import org.apache.lucene.index.SegmentReader;
|
import org.apache.lucene.index.SegmentReader;
|
||||||
|
import org.elasticsearch.common.Nullable;
|
||||||
import org.elasticsearch.index.Index;
|
import org.elasticsearch.index.Index;
|
||||||
import org.elasticsearch.index.mapper.FieldMapper;
|
import org.elasticsearch.index.mapper.FieldMapper;
|
||||||
|
|
||||||
|
@ -48,7 +49,7 @@ public interface IndexFieldDataCache {
|
||||||
|
|
||||||
void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);
|
void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);
|
||||||
|
|
||||||
void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, RemovalNotification<Object, AtomicFieldData> notification);
|
void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, @Nullable AtomicFieldData fieldData);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -72,7 +73,7 @@ public interface IndexFieldDataCache {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onRemoval(RemovalNotification<Object, AtomicFieldData> notification) {
|
public void onRemoval(RemovalNotification<Object, AtomicFieldData> notification) {
|
||||||
listener.onUnload(index, fieldNames, fieldDataType, notification);
|
listener.onUnload(index, fieldNames, fieldDataType, notification.wasEvicted(), notification.getValue());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -19,10 +19,10 @@
|
||||||
|
|
||||||
package org.elasticsearch.index.fielddata;
|
package org.elasticsearch.index.fielddata;
|
||||||
|
|
||||||
import com.google.common.cache.RemovalNotification;
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import org.apache.lucene.index.IndexReader;
|
import org.apache.lucene.index.IndexReader;
|
||||||
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
||||||
|
import org.elasticsearch.common.Nullable;
|
||||||
import org.elasticsearch.common.collect.MapBuilder;
|
import org.elasticsearch.common.collect.MapBuilder;
|
||||||
import org.elasticsearch.common.collect.Tuple;
|
import org.elasticsearch.common.collect.Tuple;
|
||||||
import org.elasticsearch.common.inject.Inject;
|
import org.elasticsearch.common.inject.Inject;
|
||||||
|
@ -35,6 +35,7 @@ import org.elasticsearch.index.Index;
|
||||||
import org.elasticsearch.index.fielddata.plain.*;
|
import org.elasticsearch.index.fielddata.plain.*;
|
||||||
import org.elasticsearch.index.mapper.FieldMapper;
|
import org.elasticsearch.index.mapper.FieldMapper;
|
||||||
import org.elasticsearch.index.settings.IndexSettings;
|
import org.elasticsearch.index.settings.IndexSettings;
|
||||||
|
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
|
||||||
|
|
||||||
import java.util.concurrent.ConcurrentMap;
|
import java.util.concurrent.ConcurrentMap;
|
||||||
|
|
||||||
|
@ -70,18 +71,20 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
|
||||||
.immutableMap();
|
.immutableMap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private final IndicesFieldDataCache indicesFieldDataCache;
|
||||||
private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
|
private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
|
||||||
|
|
||||||
private final CounterMetric memoryUsedInBytes = new CounterMetric();
|
private final CounterMetric memoryUsedInBytes = new CounterMetric();
|
||||||
private final CounterMetric evictions = new CounterMetric();
|
private final CounterMetric evictions = new CounterMetric();
|
||||||
|
|
||||||
public IndexFieldDataService(Index index) {
|
public IndexFieldDataService(Index index) {
|
||||||
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
|
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, new IndicesFieldDataCache(ImmutableSettings.Builder.EMPTY_SETTINGS));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
public IndexFieldDataService(Index index, @IndexSettings Settings indexSettings) {
|
public IndexFieldDataService(Index index, @IndexSettings Settings indexSettings, IndicesFieldDataCache indicesFieldDataCache) {
|
||||||
super(index, indexSettings);
|
super(index, indexSettings);
|
||||||
|
this.indicesFieldDataCache = indicesFieldDataCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void clear() {
|
public void clear() {
|
||||||
|
@ -115,12 +118,12 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, RemovalNotification<Object, AtomicFieldData> notification) {
|
public void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, @Nullable AtomicFieldData fieldData) {
|
||||||
assert index.equals(this.index);
|
assert index.equals(this.index);
|
||||||
if (notification.getValue() != null) {
|
if (fieldData != null) {
|
||||||
memoryUsedInBytes.dec(notification.getValue().getMemorySizeInBytes());
|
memoryUsedInBytes.dec(fieldData.getMemorySizeInBytes());
|
||||||
}
|
}
|
||||||
if (notification.wasEvicted()) {
|
if (wasEvicted) {
|
||||||
evictions.inc();
|
evictions.inc();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -160,6 +163,8 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
|
||||||
cache = new IndexFieldDataCache.Resident(index, fieldNames, type, this);
|
cache = new IndexFieldDataCache.Resident(index, fieldNames, type, this);
|
||||||
} else if ("soft".equals(cacheType)) {
|
} else if ("soft".equals(cacheType)) {
|
||||||
cache = new IndexFieldDataCache.Soft(index, fieldNames, type, this);
|
cache = new IndexFieldDataCache.Soft(index, fieldNames, type, this);
|
||||||
|
} else if ("node".equals(cacheType)) {
|
||||||
|
cache = indicesFieldDataCache.buildIndexFieldDataCache(index, fieldNames, type, this);
|
||||||
} else {
|
} else {
|
||||||
throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
|
throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,6 +28,7 @@ import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
|
||||||
import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
|
import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
|
||||||
import org.elasticsearch.indices.cache.filter.terms.IndicesTermsFilterCache;
|
import org.elasticsearch.indices.cache.filter.terms.IndicesTermsFilterCache;
|
||||||
import org.elasticsearch.indices.cluster.IndicesClusterStateService;
|
import org.elasticsearch.indices.cluster.IndicesClusterStateService;
|
||||||
|
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
|
||||||
import org.elasticsearch.indices.memory.IndexingMemoryController;
|
import org.elasticsearch.indices.memory.IndexingMemoryController;
|
||||||
import org.elasticsearch.indices.query.IndicesQueriesModule;
|
import org.elasticsearch.indices.query.IndicesQueriesModule;
|
||||||
import org.elasticsearch.indices.recovery.RecoverySettings;
|
import org.elasticsearch.indices.recovery.RecoverySettings;
|
||||||
|
@ -69,6 +70,7 @@ public class IndicesModule extends AbstractModule implements SpawnModules {
|
||||||
bind(IndicesClusterStateService.class).asEagerSingleton();
|
bind(IndicesClusterStateService.class).asEagerSingleton();
|
||||||
bind(IndexingMemoryController.class).asEagerSingleton();
|
bind(IndexingMemoryController.class).asEagerSingleton();
|
||||||
bind(IndicesFilterCache.class).asEagerSingleton();
|
bind(IndicesFilterCache.class).asEagerSingleton();
|
||||||
|
bind(IndicesFieldDataCache.class).asEagerSingleton();
|
||||||
bind(IndicesTermsFilterCache.class).asEagerSingleton();
|
bind(IndicesTermsFilterCache.class).asEagerSingleton();
|
||||||
bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton();
|
bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton();
|
||||||
bind(IndicesTTLService.class).asEagerSingleton();
|
bind(IndicesTTLService.class).asEagerSingleton();
|
||||||
|
|
202
src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
vendored
Normal file
202
src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
vendored
Normal file
|
@ -0,0 +1,202 @@
|
||||||
|
/*
|
||||||
|
* Licensed to ElasticSearch and Shay Banon under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. ElasticSearch licenses this
|
||||||
|
* file to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.elasticsearch.indices.fielddata.cache;
|
||||||
|
|
||||||
|
import com.google.common.cache.*;
|
||||||
|
import org.apache.lucene.index.AtomicReaderContext;
|
||||||
|
import org.apache.lucene.index.IndexReader;
|
||||||
|
import org.apache.lucene.index.SegmentReader;
|
||||||
|
import org.elasticsearch.common.component.AbstractComponent;
|
||||||
|
import org.elasticsearch.common.inject.Inject;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.index.Index;
|
||||||
|
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||||
|
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||||
|
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||||
|
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
|
||||||
|
import org.elasticsearch.index.mapper.FieldMapper;
|
||||||
|
import org.elasticsearch.monitor.jvm.JvmInfo;
|
||||||
|
|
||||||
|
import java.util.concurrent.Callable;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
*/
|
||||||
|
public class IndicesFieldDataCache extends AbstractComponent implements RemovalListener<IndicesFieldDataCache.Key, AtomicFieldData> {
|
||||||
|
|
||||||
|
Cache<Key, AtomicFieldData> cache;
|
||||||
|
|
||||||
|
private volatile String size;
|
||||||
|
private volatile long sizeInBytes;
|
||||||
|
private volatile TimeValue expire;
|
||||||
|
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
public IndicesFieldDataCache(Settings settings) {
|
||||||
|
super(settings);
|
||||||
|
this.size = componentSettings.get("size", "40%");
|
||||||
|
this.expire = componentSettings.getAsTime("expire", null);
|
||||||
|
computeSizeInBytes();
|
||||||
|
buildCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void buildCache() {
|
||||||
|
CacheBuilder<Key, AtomicFieldData> cacheBuilder = CacheBuilder.newBuilder()
|
||||||
|
.removalListener(this);
|
||||||
|
cacheBuilder.maximumWeight(sizeInBytes).weigher(new FieldDataWeigher());
|
||||||
|
// defaults to 4, but this is a busy map for all indices, increase it a bit
|
||||||
|
cacheBuilder.concurrencyLevel(16);
|
||||||
|
if (expire != null) {
|
||||||
|
cacheBuilder.expireAfterAccess(expire.millis(), TimeUnit.MILLISECONDS);
|
||||||
|
}
|
||||||
|
cache = cacheBuilder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void computeSizeInBytes() {
|
||||||
|
if (size.endsWith("%")) {
|
||||||
|
double percent = Double.parseDouble(size.substring(0, size.length() - 1));
|
||||||
|
sizeInBytes = (long) ((percent / 100) * JvmInfo.jvmInfo().getMem().getHeapMax().bytes());
|
||||||
|
} else {
|
||||||
|
sizeInBytes = ByteSizeValue.parseBytesSizeValue(size).bytes();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close() {
|
||||||
|
cache.invalidateAll();
|
||||||
|
}
|
||||||
|
|
||||||
|
public IndexFieldDataCache buildIndexFieldDataCache(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache.Listener listener) {
|
||||||
|
return new IndexFieldCache(index, fieldNames, fieldDataType, listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onRemoval(RemovalNotification<Key, AtomicFieldData> notification) {
|
||||||
|
if (notification.getKey() != null) {
|
||||||
|
IndexFieldCache indexFieldCache = notification.getKey().indexCache;
|
||||||
|
indexFieldCache.listener.onUnload(indexFieldCache.index, indexFieldCache.fieldNames, indexFieldCache.fieldDataType, notification.wasEvicted(), notification.getValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class FieldDataWeigher implements Weigher<Key, AtomicFieldData> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int weigh(Key key, AtomicFieldData fieldData) {
|
||||||
|
int weight = (int) Math.min(fieldData.getMemorySizeInBytes(), Integer.MAX_VALUE);
|
||||||
|
return weight == 0 ? 1 : weight;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A specific cache instance for the relevant parameters of it (index, fieldNames, fieldType).
|
||||||
|
*/
|
||||||
|
class IndexFieldCache implements IndexFieldDataCache, SegmentReader.CoreClosedListener {
|
||||||
|
|
||||||
|
final Index index;
|
||||||
|
final FieldMapper.Names fieldNames;
|
||||||
|
final FieldDataType fieldDataType;
|
||||||
|
final Listener listener;
|
||||||
|
|
||||||
|
IndexFieldCache(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
|
||||||
|
this.index = index;
|
||||||
|
this.fieldNames = fieldNames;
|
||||||
|
this.fieldDataType = fieldDataType;
|
||||||
|
this.listener = listener;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
|
||||||
|
Key key = new Key(this, context.reader().getCoreCacheKey());
|
||||||
|
//noinspection unchecked
|
||||||
|
return (FD) cache.get(key, new Callable<AtomicFieldData>() {
|
||||||
|
@Override
|
||||||
|
public AtomicFieldData call() throws Exception {
|
||||||
|
if (context.reader() instanceof SegmentReader) {
|
||||||
|
((SegmentReader) context.reader()).addCoreClosedListener(IndexFieldCache.this);
|
||||||
|
}
|
||||||
|
AtomicFieldData fieldData = indexFieldData.loadDirect(context);
|
||||||
|
listener.onLoad(index, indexFieldData.getFieldNames(), fieldDataType, fieldData);
|
||||||
|
return fieldData;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onClose(SegmentReader owner) {
|
||||||
|
cache.invalidate(new Key(this, owner.getCoreCacheKey()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void clear(Index index) {
|
||||||
|
for (Key key : cache.asMap().keySet()) {
|
||||||
|
if (key.indexCache.index.equals(index)) {
|
||||||
|
cache.invalidate(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void clear(Index index, String fieldName) {
|
||||||
|
for (Key key : cache.asMap().keySet()) {
|
||||||
|
if (key.indexCache.index.equals(index)) {
|
||||||
|
if (key.indexCache.fieldNames.fullName().equals(fieldName)) {
|
||||||
|
cache.invalidate(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void clear(Index index, IndexReader reader) {
|
||||||
|
cache.invalidate(new Key(this, reader.getCoreCacheKey()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class Key {
|
||||||
|
public final IndexFieldCache indexCache;
|
||||||
|
public final Object readerKey;
|
||||||
|
|
||||||
|
Key(IndexFieldCache indexCache, Object readerKey) {
|
||||||
|
this.indexCache = indexCache;
|
||||||
|
this.readerKey = readerKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) return true;
|
||||||
|
if (o == null || getClass() != o.getClass()) return false;
|
||||||
|
|
||||||
|
Key key = (Key) o;
|
||||||
|
|
||||||
|
if (!indexCache.equals(key.indexCache)) return false;
|
||||||
|
if (!readerKey.equals(key.readerKey)) return false;
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
int result = indexCache.hashCode();
|
||||||
|
result = 31 * result + readerKey.hashCode();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -67,6 +67,7 @@ import org.elasticsearch.indices.IndicesModule;
|
||||||
import org.elasticsearch.indices.IndicesService;
|
import org.elasticsearch.indices.IndicesService;
|
||||||
import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
|
import org.elasticsearch.indices.cache.filter.IndicesFilterCache;
|
||||||
import org.elasticsearch.indices.cluster.IndicesClusterStateService;
|
import org.elasticsearch.indices.cluster.IndicesClusterStateService;
|
||||||
|
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
|
||||||
import org.elasticsearch.indices.memory.IndexingMemoryController;
|
import org.elasticsearch.indices.memory.IndexingMemoryController;
|
||||||
import org.elasticsearch.indices.ttl.IndicesTTLService;
|
import org.elasticsearch.indices.ttl.IndicesTTLService;
|
||||||
import org.elasticsearch.jmx.JmxModule;
|
import org.elasticsearch.jmx.JmxModule;
|
||||||
|
@ -297,6 +298,7 @@ public final class InternalNode implements Node {
|
||||||
injector.getInstance(IndicesClusterStateService.class).close();
|
injector.getInstance(IndicesClusterStateService.class).close();
|
||||||
stopWatch.stop().start("indices");
|
stopWatch.stop().start("indices");
|
||||||
injector.getInstance(IndicesFilterCache.class).close();
|
injector.getInstance(IndicesFilterCache.class).close();
|
||||||
|
injector.getInstance(IndicesFieldDataCache.class).close();
|
||||||
injector.getInstance(IndexingMemoryController.class).close();
|
injector.getInstance(IndexingMemoryController.class).close();
|
||||||
injector.getInstance(IndicesTTLService.class).close();
|
injector.getInstance(IndicesTTLService.class).close();
|
||||||
injector.getInstance(IndicesService.class).close();
|
injector.getInstance(IndicesService.class).close();
|
||||||
|
|
Loading…
Reference in New Issue