diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java index 3e8f041d4f8..57671f86c04 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/mapping/delete/TransportDeleteMappingAction.java @@ -22,6 +22,8 @@ package org.elasticsearch.action.admin.indices.mapping.delete; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.TransportActions; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.refresh.TransportRefreshAction; import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; import org.elasticsearch.action.deletebyquery.TransportDeleteByQueryAction; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; @@ -52,11 +54,15 @@ public class TransportDeleteMappingAction extends TransportMasterNodeOperationAc private final TransportDeleteByQueryAction deleteByQueryAction; + private final TransportRefreshAction refreshAction; + @Inject public TransportDeleteMappingAction(Settings settings, TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, MetaDataMappingService metaDataMappingService, TransportDeleteByQueryAction deleteByQueryAction) { + ThreadPool threadPool, MetaDataMappingService metaDataMappingService, + TransportDeleteByQueryAction deleteByQueryAction, TransportRefreshAction refreshAction) { super(settings, transportService, clusterService, threadPool); this.metaDataMappingService = metaDataMappingService; this.deleteByQueryAction = deleteByQueryAction; + this.refreshAction = refreshAction; } @@ -87,8 +93,17 @@ public class TransportDeleteMappingAction extends TransportMasterNodeOperationAc final CountDownLatch latch = new CountDownLatch(1); deleteByQueryAction.execute(Requests.deleteByQueryRequest(request.indices()).query(QueryBuilders.filtered(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(TypeFieldMapper.NAME, request.type()))), new ActionListener() { @Override public void onResponse(DeleteByQueryResponse deleteByQueryResponse) { - metaDataMappingService.removeMapping(new MetaDataMappingService.RemoveRequest(request.indices(), request.type())); - latch.countDown(); + refreshAction.execute(Requests.refreshRequest(request.indices()), new ActionListener() { + @Override public void onResponse(RefreshResponse refreshResponse) { + metaDataMappingService.removeMapping(new MetaDataMappingService.RemoveRequest(request.indices(), request.type())); + latch.countDown(); + } + + @Override public void onFailure(Throwable e) { + metaDataMappingService.removeMapping(new MetaDataMappingService.RemoveRequest(request.indices(), request.type())); + latch.countDown(); + } + }); } @Override public void onFailure(Throwable e) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/collect/Tuple.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/collect/Tuple.java index 95287629512..5d570d6253b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/collect/Tuple.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/collect/Tuple.java @@ -43,4 +43,22 @@ public class Tuple { public V2 v2() { return v2; } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Tuple tuple = (Tuple) o; + + if (v1 != null ? !v1.equals(tuple.v1) : tuple.v1 != null) return false; + if (v2 != null ? !v2.equals(tuple.v2) : tuple.v2 != null) return false; + + return true; + } + + @Override public int hashCode() { + int result = v1 != null ? v1.hashCode() : 0; + result = 31 * result + (v2 != null ? v2.hashCode() : 0); + return result; + } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java index 4a2224e2a91..fb45a638161 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/soft/SoftFieldDataCache.java @@ -35,9 +35,11 @@ import java.util.concurrent.ConcurrentMap; public class SoftFieldDataCache extends AbstractConcurrentMapFieldDataCache { @Inject public SoftFieldDataCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings, new MapMaker() - .softKeys() - .>makeMap()); + super(index, indexSettings); + } + + @Override protected ConcurrentMap buildFieldDataMap() { + return new MapMaker().softValues().makeMap(); } @Override public String type() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java index 9e5738091a8..46165f5a68c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.cache.field.data.support; import org.apache.lucene.index.IndexReader; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.collect.MapMaker; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.AbstractIndexComponent; @@ -41,10 +42,11 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC private final Object creationMutex = new Object(); - protected AbstractConcurrentMapFieldDataCache(Index index, @IndexSettings Settings indexSettings, - ConcurrentMap> cache) { + protected AbstractConcurrentMapFieldDataCache(Index index, @IndexSettings Settings indexSettings) { super(index, indexSettings); - this.cache = cache; + // weak keys is fine, it will only be cleared once IndexReader references will be removed + // (assuming clear(...) will not be called) + this.cache = new MapMaker().weakKeys().makeMap(); } @Override public void close() throws ElasticSearchException { @@ -56,7 +58,11 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC } @Override public void clear(IndexReader reader) { - cache.remove(reader.getFieldCacheKey()); + ConcurrentMap map = cache.remove(reader.getFieldCacheKey()); + // help soft/weak handling GC + if (map != null) { + map.clear(); + } } @Override public void clearUnreferenced() { @@ -67,7 +73,7 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC return cache(type.fieldDataClass(), reader, fieldName); } - protected ConcurrentMap buildFilterMap() { + protected ConcurrentMap buildFieldDataMap() { return ConcurrentCollections.newConcurrentMap(); } @@ -77,7 +83,7 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC synchronized (creationMutex) { fieldDataCache = cache.get(reader.getFieldCacheKey()); if (fieldDataCache == null) { - fieldDataCache = buildFilterMap(); + fieldDataCache = buildFieldDataMap(); cache.put(reader.getFieldCacheKey(), fieldDataCache); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java index 55680dbd117..48362426476 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/weak/WeakFieldDataCache.java @@ -35,9 +35,11 @@ import java.util.concurrent.ConcurrentMap; public class WeakFieldDataCache extends AbstractConcurrentMapFieldDataCache { @Inject public WeakFieldDataCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings, new MapMaker() - .weakKeys() - .>makeMap()); + super(index, indexSettings); + } + + @Override protected ConcurrentMap buildFieldDataMap() { + return new MapMaker().weakValues().makeMap(); } @Override public String type() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java index d564d79b260..d30d1995093 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/FilterCacheModule.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.cache.filter; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Scopes; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.cache.filter.weak.WeakFilterCache; +import org.elasticsearch.index.cache.filter.soft.SoftFilterCache; /** * @author kimchy (Shay Banon) @@ -41,7 +41,7 @@ public class FilterCacheModule extends AbstractModule { @Override protected void configure() { bind(FilterCache.class) - .to(settings.getAsClass(FilterCacheSettings.FILTER_CACHE_TYPE, WeakFilterCache.class, "org.elasticsearch.index.cache.filter.", "FilterCache")) + .to(settings.getAsClass(FilterCacheSettings.FILTER_CACHE_TYPE, SoftFilterCache.class, "org.elasticsearch.index.cache.filter.", "FilterCache")) .in(Scopes.SINGLETON); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java index 423c849d7d9..42869a7c8ab 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/soft/SoftFilterCache.java @@ -38,7 +38,13 @@ import java.util.concurrent.ConcurrentMap; public class SoftFilterCache extends AbstractConcurrentMapFilterCache { @Inject public SoftFilterCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings, new MapMaker().softKeys().>makeMap()); + super(index, indexSettings); + } + + @Override protected ConcurrentMap buildFilterMap() { + // DocSet are not really stored with strong reference only when searching on them... + // Filter might be stored in query cache + return new MapMaker().softValues().makeMap(); } @Override public String type() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java index 38b60501d33..fb03d4f5cc5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.cache.filter.support; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; +import org.elasticsearch.common.collect.MapMaker; import org.elasticsearch.common.lucene.docset.DocSet; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; @@ -44,10 +45,11 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp final ConcurrentMap> cache; - protected AbstractConcurrentMapFilterCache(Index index, @IndexSettings Settings indexSettings, - ConcurrentMap> cache) { + protected AbstractConcurrentMapFilterCache(Index index, @IndexSettings Settings indexSettings) { super(index, indexSettings); - this.cache = cache; + // weak keys is fine, it will only be cleared once IndexReader references will be removed + // (assuming clear(...) will not be called) + this.cache = new MapMaker().weakKeys().makeMap(); } @Override public void close() { @@ -59,7 +61,11 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp } @Override public void clear(IndexReader reader) { - cache.remove(reader.getFieldCacheKey()); + ConcurrentMap map = cache.remove(reader.getFieldCacheKey()); + // help soft/weak handling GC + if (map != null) { + map.clear(); + } } @Override public void clearUnreferenced() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java index 757c1ebbf0f..4975cdb625f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/weak/WeakFilterCache.java @@ -38,7 +38,13 @@ import java.util.concurrent.ConcurrentMap; public class WeakFilterCache extends AbstractConcurrentMapFilterCache { @Inject public WeakFilterCache(Index index, @IndexSettings Settings indexSettings) { - super(index, indexSettings, new MapMaker().weakKeys().>makeMap()); + super(index, indexSettings); + } + + @Override protected ConcurrentMap buildFilterMap() { + // DocSet are not really stored with strong reference only when searching on them... + // Filter might be stored in query cache + return new MapMaker().weakValues().makeMap(); } @Override public String type() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index b4bdb1e7d81..998adb150df 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -35,11 +35,13 @@ import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.collect.ImmutableMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.IndexShardAlreadyExistsException; import org.elasticsearch.index.IndexShardMissingException; import org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException; @@ -58,6 +60,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import static org.elasticsearch.ExceptionsHelper.*; import static org.elasticsearch.common.collect.Sets.*; @@ -83,6 +86,9 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent, Boolean> seenMappings = ConcurrentCollections.newConcurrentMap(); + @Inject public IndicesClusterStateService(Settings settings, IndicesService indicesService, ClusterService clusterService, ThreadPool threadPool, RecoveryTarget recoveryTarget, ShardStateAction shardStateAction, NodeIndexCreatedAction nodeIndexCreatedAction, NodeIndexDeletedAction nodeIndexDeletedAction, @@ -199,6 +205,9 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent entry : mappings.entrySet()) { String mappingType = entry.getKey(); CompressedString mappingSource = entry.getValue(); + if (!seenMappings.containsKey(new Tuple(index, mappingType))) { + seenMappings.put(new Tuple(index, mappingType), true); + } try { if (!mapperService.hasMapping(mappingType)) { @@ -224,8 +233,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent(index, documentMapper.type())) && !mappings.containsKey(documentMapper.type())) { + // we have it in our mappings, but not in the metadata, and we have seen it in the cluster state, remove it mapperService.remove(documentMapper.type()); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java index 91d82d754ed..15bad8ee2f4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java @@ -54,8 +54,6 @@ import org.elasticsearch.gateway.GatewayModule; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.http.HttpServer; import org.elasticsearch.http.HttpServerModule; -import org.elasticsearch.indexer.IndexerManager; -import org.elasticsearch.indexer.IndexersModule; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.cluster.IndicesClusterStateService; @@ -134,7 +132,7 @@ public final class InternalNode implements Node { if (settings.getAsBoolean("http.enabled", true)) { modules.add(new HttpServerModule(settings)); } - modules.add(new IndexersModule(settings)); +// modules.add(new IndexersModule(settings)); modules.add(new IndicesModule(settings)); modules.add(new SearchModule()); modules.add(new TransportActionModule()); @@ -171,7 +169,7 @@ public final class InternalNode implements Node { injector.getInstance(IndicesService.class).start(); injector.getInstance(IndicesClusterStateService.class).start(); - injector.getInstance(IndexerManager.class).start(); +// injector.getInstance(IndexerManager.class).start(); injector.getInstance(ClusterService.class).start(); injector.getInstance(RoutingService.class).start(); injector.getInstance(SearchService.class).start(); @@ -209,7 +207,7 @@ public final class InternalNode implements Node { injector.getInstance(MonitorService.class).stop(); injector.getInstance(GatewayService.class).stop(); injector.getInstance(SearchService.class).stop(); - injector.getInstance(IndexerManager.class).stop(); +// injector.getInstance(IndexerManager.class).stop(); injector.getInstance(IndicesClusterStateService.class).stop(); injector.getInstance(IndicesService.class).stop(); injector.getInstance(RestController.class).stop(); @@ -256,7 +254,7 @@ public final class InternalNode implements Node { stopWatch.stop().start("search"); injector.getInstance(SearchService.class).close(); stopWatch.stop().start("indexers"); - injector.getInstance(IndexerManager.class).close(); +// injector.getInstance(IndexerManager.class).close(); stopWatch.stop().start("indices_cluster"); injector.getInstance(IndicesClusterStateService.class).close(); stopWatch.stop().start("indices"); diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facets/SimpleFacetsTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facets/SimpleFacetsTests.java index 7deeb700406..ce72b6250ce 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facets/SimpleFacetsTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facets/SimpleFacetsTests.java @@ -122,6 +122,7 @@ public class SimpleFacetsTests extends AbstractNodesTests { @Test public void testTermsIndexFacet() throws Exception { try { + client.admin().indices().prepareDelete("test").execute().actionGet(); client.admin().indices().prepareDelete("test1").execute().actionGet(); client.admin().indices().prepareDelete("test2").execute().actionGet(); } catch (Exception e) {