diff --git a/core/pom.xml b/core/pom.xml index a96e3746bbd..2a5f6deabd1 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -274,7 +274,7 @@ org/elasticsearch/common/cli/CliToolTestCase$*.class org/elasticsearch/cluster/MockInternalClusterInfoService.class org/elasticsearch/cluster/MockInternalClusterInfoService$*.class - org/elasticsearch/index/shard/MockEngineFactoryPlugin.class + org/elasticsearch/index/MockEngineFactoryPlugin.class org/elasticsearch/search/MockSearchService.class org/elasticsearch/search/MockSearchService$*.class org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.class diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java index be7a3f0d4de..ff754be2a20 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.fielddata.FieldDataStats; -import org.elasticsearch.index.percolator.stats.PercolateStats; +import org.elasticsearch.index.percolator.PercolateStats; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.search.suggest.completion.CompletionStats; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java index 00579290ccc..2308d7be6af 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java @@ -83,7 +83,7 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastByNodeAc protected EmptyResult shardOperation(ClearIndicesCacheRequest request, ShardRouting shardRouting) { IndexService service = indicesService.indexService(shardRouting.getIndex()); if (service != null) { - IndexShard shard = service.shard(shardRouting.id()); + IndexShard shard = service.getShardOrNull(shardRouting.id()); boolean clearedAtLeastOne = false; if (request.queryCache()) { clearedAtLeastOne = true; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 2bae799af88..f768cfedc94 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -62,7 +62,7 @@ public class TransportShardFlushAction extends TransportReplicationAction shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { - IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).shardSafe(shardRequest.shardId.id()); + IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).getShard(shardRequest.shardId.id()); indexShard.flush(shardRequest.request.getRequest()); logger.trace("{} flush request executed on primary", indexShard.shardId()); return new Tuple<>(new ActionWriteResponse(), shardRequest.request); @@ -70,7 +70,7 @@ public class TransportShardFlushAction extends TransportReplicationAction shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { - IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).shardSafe(shardRequest.shardId.id()); + IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).getShard(shardRequest.shardId.id()); indexShard.refresh("api"); logger.trace("{} refresh request executed on primary", indexShard.shardId()); return new Tuple<>(new ActionWriteResponse(), shardRequest.request); @@ -71,7 +71,7 @@ public class TransportShardRefreshAction extends TransportReplicationAction segments = indexShard.engine().segments(false); + IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); + List segments = indexShard.segments(false); long total_bytes = 0; long to_upgrade_bytes = 0; long to_upgrade_bytes_ancient = 0; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java index 38375af66f9..30aff1f2e6e 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java @@ -119,7 +119,7 @@ public class TransportUpgradeAction extends TransportBroadcastByNodeAction shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) { final BulkShardRequest request = shardRequest.request; final IndexService indexService = indicesService.indexServiceSafe(request.index()); - final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id()); + final IndexShard indexShard = indexService.getShard(shardRequest.shardId.id()); long[] preVersions = new long[request.items().length]; VersionType[] preVersionTypes = new VersionType[request.items().length]; @@ -447,7 +447,7 @@ public class TransportShardBulkAction extends TransportReplicationAction shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) { DeleteRequest request = shardRequest.request; - IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).shardSafe(shardRequest.shardId.id()); + IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()).getShard(shardRequest.shardId.id()); Engine.Delete delete = indexShard.prepareDelete(request.type(), request.id(), request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY); indexShard.delete(delete); // update the request with teh version so it will go to the replicas @@ -146,7 +145,7 @@ public class TransportDeleteAction extends TransportReplicationAction fieldStats = new HashMap<>(); IndexService indexServices = indicesService.indexServiceSafe(shardId.getIndex()); MapperService mapperService = indexServices.mapperService(); - IndexShard shard = indexServices.shardSafe(shardId.id()); + IndexShard shard = indexServices.getShard(shardId.id()); try (Engine.Searcher searcher = shard.acquireSearcher("fieldstats")) { for (String field : request.getFields()) { MappedFieldType fieldType = mapperService.fullName(field); diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index cba68bd281f..0bcadd6c90a 100644 --- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -92,7 +92,7 @@ public class TransportGetAction extends TransportSingleShardAction result = executeIndexRequestOnPrimary(null, request, indexShard); final IndexResponse response = result.response; @@ -176,7 +176,7 @@ public class TransportIndexAction extends TransportReplicationAction listener, final int retryCount) { IndexService indexService = indicesService.indexServiceSafe(request.concreteIndex()); - IndexShard indexShard = indexService.shardSafe(request.shardId()); + IndexShard indexShard = indexService.getShard(request.shardId()); final UpdateHelper.Result result = updateHelper.prepare(request, indexShard); switch (result.operation()) { case UPSERT: @@ -266,7 +266,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio UpdateResponse update = result.action(); IndexService indexServiceOrNull = indicesService.indexService(request.concreteIndex()); if (indexServiceOrNull != null) { - IndexShard shard = indexService.shard(request.shardId()); + IndexShard shard = indexService.getShardOrNull(request.shardId()); if (shard != null) { shard.indexingService().noopUpdate(request.type()); } diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java index d94eb4f9c7d..0c70dd456ca 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java @@ -20,21 +20,31 @@ package org.elasticsearch.index; import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.inject.util.Providers; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.shard.IndexSearcherWrapper; /** * */ public class IndexModule extends AbstractModule { - private final Settings settings; - - public IndexModule(Settings settings) { - this.settings = settings; - } - + // pkg private so tests can mock + Class engineFactoryImpl = InternalEngineFactory.class; + Class indexSearcherWrapper = null; + @Override protected void configure() { + bind(EngineFactory.class).to(engineFactoryImpl).asEagerSingleton(); + if (indexSearcherWrapper == null) { + bind(IndexSearcherWrapper.class).toProvider(Providers.of(null)); + } else { + bind(IndexSearcherWrapper.class).to(indexSearcherWrapper).asEagerSingleton(); + } bind(IndexService.class).asEagerSingleton(); + bind(IndexServicesProvider.class).asEagerSingleton(); } + + } diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index 3c40b02a4b9..2fc7a242db1 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -24,16 +24,9 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.inject.CreationException; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.Injectors; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; @@ -49,20 +42,12 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.settings.IndexSettingsService; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.IndexShardModule; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardNotFoundException; -import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.shard.*; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.StoreModule; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InternalIndicesLifecycle; -import org.elasticsearch.indices.cache.query.IndicesQueryCache; -import org.elasticsearch.plugins.PluginsService; import java.io.Closeable; import java.io.IOException; @@ -81,86 +66,42 @@ import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; */ public class IndexService extends AbstractIndexComponent implements IndexComponent, Iterable { - private final Injector injector; - private final Settings indexSettings; - - private final PluginsService pluginsService; - private final InternalIndicesLifecycle indicesLifecycle; - private final AnalysisService analysisService; - - private final MapperService mapperService; - - private final IndexQueryParserService queryParserService; - - private final SimilarityService similarityService; - - private final IndexAliasesService aliasesService; - - private final IndexCache indexCache; - private final IndexFieldDataService indexFieldData; - private final BitsetFilterCache bitsetFilterCache; - private final IndexSettingsService settingsService; - private final NodeEnvironment nodeEnv; private final IndicesService indicesServices; - - private volatile ImmutableMap shards = ImmutableMap.of(); - - private static class IndexShardInjectorPair { - private final IndexShard indexShard; - private final Injector injector; - - public IndexShardInjectorPair(IndexShard indexShard, Injector injector) { - this.indexShard = indexShard; - this.injector = injector; - } - - public IndexShard getIndexShard() { - return indexShard; - } - - public Injector getInjector() { - return injector; - } - } - + private final IndexServicesProvider indexServicesProvider; + private final IndexStore indexStore; + private volatile ImmutableMap shards = ImmutableMap.of(); private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean deleted = new AtomicBoolean(false); @Inject - public IndexService(Injector injector, Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv, - AnalysisService analysisService, MapperService mapperService, IndexQueryParserService queryParserService, - SimilarityService similarityService, IndexAliasesService aliasesService, IndexCache indexCache, + public IndexService(Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv, + AnalysisService analysisService, IndexSettingsService settingsService, - IndexFieldDataService indexFieldData, BitsetFilterCache bitSetFilterCache, IndicesService indicesServices) { - + IndexFieldDataService indexFieldData, + BitsetFilterCache bitSetFilterCache, + IndicesService indicesServices, + IndexServicesProvider indexServicesProvider, + IndexStore indexStore) { super(index, indexSettings); - this.injector = injector; this.indexSettings = indexSettings; this.analysisService = analysisService; - this.mapperService = mapperService; - this.queryParserService = queryParserService; - this.similarityService = similarityService; - this.aliasesService = aliasesService; - this.indexCache = indexCache; this.indexFieldData = indexFieldData; this.settingsService = settingsService; this.bitsetFilterCache = bitSetFilterCache; - - this.pluginsService = injector.getInstance(PluginsService.class); this.indicesServices = indicesServices; - this.indicesLifecycle = (InternalIndicesLifecycle) injector.getInstance(IndicesLifecycle.class); - - // inject workarounds for cyclic dep + this.indicesLifecycle = (InternalIndicesLifecycle) indexServicesProvider.getIndicesLifecycle(); + this.nodeEnv = nodeEnv; + this.indexServicesProvider = indexServicesProvider; + this.indexStore = indexStore; indexFieldData.setListener(new FieldDataCacheListener(this)); bitSetFilterCache.setListener(new BitsetCacheListener(this)); - this.nodeEnv = nodeEnv; } public int numberOfShards() { @@ -173,7 +114,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone @Override public Iterator iterator() { - return shards.values().stream().map((p) -> p.getIndexShard()).iterator(); + return shards.values().iterator(); } public boolean hasShard(int shardId) { @@ -184,19 +125,15 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone * Return the shard with the provided id, or null if there is no such shard. */ @Nullable - public IndexShard shard(int shardId) { - IndexShardInjectorPair indexShardInjectorPair = shards.get(shardId); - if (indexShardInjectorPair != null) { - return indexShardInjectorPair.getIndexShard(); - } - return null; + public IndexShard getShardOrNull(int shardId) { + return shards.get(shardId); } /** * Return the shard with the provided id, or throw an exception if it doesn't exist. */ - public IndexShard shardSafe(int shardId) { - IndexShard indexShard = shard(shardId); + public IndexShard getShard(int shardId) { + IndexShard indexShard = getShardOrNull(shardId); if (indexShard == null) { throw new ShardNotFoundException(new ShardId(index, shardId)); } @@ -207,16 +144,12 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone return shards.keySet(); } - public Injector injector() { - return injector; - } - public IndexSettingsService settingsService() { return this.settingsService; } public IndexCache cache() { - return indexCache; + return indexServicesProvider.getIndexCache(); } public IndexFieldDataService fieldData() { @@ -232,19 +165,19 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } public MapperService mapperService() { - return mapperService; + return indexServicesProvider.getMapperService(); } public IndexQueryParserService queryParserService() { - return queryParserService; + return indexServicesProvider.getQueryParserService(); } public SimilarityService similarityService() { - return similarityService; + return indexServicesProvider.getSimilarityService(); } public IndexAliasesService aliasesService() { - return aliasesService; + return indexServicesProvider.getIndexAliasesService(); } public synchronized void close(final String reason, boolean delete) { @@ -261,16 +194,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } - /** - * Return the shard injector for the provided id, or throw an exception if there is no such shard. - */ - public Injector shardInjectorSafe(int shardId) { - IndexShardInjectorPair indexShardInjectorPair = shards.get(shardId); - if (indexShardInjectorPair == null) { - throw new ShardNotFoundException(new ShardId(index, shardId)); - } - return indexShardInjectorPair.getInjector(); - } public String indexUUID() { return indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE); @@ -301,10 +224,14 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone if (closed.get()) { throw new IllegalStateException("Can't create shard [" + index.name() + "][" + sShardId + "], closed"); } + if (indexSettings.get("index.translog.type") != null) { // TODO remove? + throw new IllegalStateException("a custom translog type is no longer supported. got [" + indexSettings.get("index.translog.type") + "]"); + } final ShardId shardId = new ShardId(index, sShardId); ShardLock lock = null; boolean success = false; - Injector shardInjector = null; + Store store = null; + IndexShard indexShard = null; try { lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5)); indicesLifecycle.beforeIndexShardCreated(shardId, indexSettings); @@ -325,7 +252,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone if (path == null) { // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard // that's being relocated/replicated we know how large it will become once it's done copying: - // Count up how many shards are currently on each data path: Map dataPathToShardCount = new HashMap<>(); for(IndexShard shard : this) { @@ -351,39 +277,17 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary. final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false || (primary && IndexMetaData.isOnSharedFilesystem(indexSettings)); - ModulesBuilder modules = new ModulesBuilder(); - // plugin modules must be added here, before others or we can get crazy injection errors... - for (Module pluginModule : pluginsService.shardModules(indexSettings)) { - modules.add(pluginModule); - } - modules.add(new IndexShardModule(shardId, primary, indexSettings)); - modules.add(new StoreModule(injector.getInstance(IndexStore.class).shardDirectory(), lock, - new StoreCloseListener(shardId, canDeleteShardContent, new Closeable() { - @Override - public void close() throws IOException { - injector.getInstance(IndicesQueryCache.class).onClose(shardId); - } - }), path)); - pluginsService.processModules(modules); - - try { - shardInjector = modules.createChildInjector(injector); - } catch (CreationException e) { - ElasticsearchException ex = new ElasticsearchException("failed to create shard", Injectors.getFirstErrorFailure(e)); - ex.setShard(shardId); - throw ex; - } catch (Throwable e) { - ElasticsearchException ex = new ElasticsearchException("failed to create shard", e); - ex.setShard(shardId); - throw ex; + store = new Store(shardId, indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> indexServicesProvider.getIndicesQueryCache().onClose(shardId))); + if (useShadowEngine(primary, indexSettings)) { + indexShard = new ShadowIndexShard(shardId, indexSettings, path, store, indexServicesProvider); + } else { + indexShard = new IndexShard(shardId, indexSettings, path, store, indexServicesProvider); } - IndexShard indexShard = shardInjector.getInstance(IndexShard.class); indicesLifecycle.indexShardStateChanged(indexShard, null, "shard created"); indicesLifecycle.afterIndexShardCreated(indexShard); - - shards = newMapBuilder(shards).put(shardId.id(), new IndexShardInjectorPair(indexShard, shardInjector)).immutableMap(); settingsService.addListener(indexShard); + shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap(); success = true; return indexShard; } catch (IOException e) { @@ -393,45 +297,35 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } finally { if (success == false) { IOUtils.closeWhileHandlingException(lock); - if (shardInjector != null) { - IndexShard indexShard = shardInjector.getInstance(IndexShard.class); - closeShardInjector("initialization failed", shardId, shardInjector, indexShard); - } + closeShard("initialization failed", shardId, indexShard, store); } } } + static boolean useShadowEngine(boolean primary, Settings indexSettings) { + return primary == false && IndexMetaData.isIndexUsingShadowReplicas(indexSettings); + } + public synchronized void removeShard(int shardId, String reason) { final ShardId sId = new ShardId(index, shardId); - final Injector shardInjector; final IndexShard indexShard; if (shards.containsKey(shardId) == false) { return; } logger.debug("[{}] closing... (reason: [{}])", shardId, reason); - HashMap tmpShardsMap = new HashMap<>(shards); - IndexShardInjectorPair indexShardInjectorPair = tmpShardsMap.remove(shardId); - indexShard = indexShardInjectorPair.getIndexShard(); - shardInjector = indexShardInjectorPair.getInjector(); + HashMap tmpShardsMap = new HashMap<>(shards); + indexShard = tmpShardsMap.remove(shardId); shards = ImmutableMap.copyOf(tmpShardsMap); - closeShardInjector(reason, sId, shardInjector, indexShard); + closeShard(reason, sId, indexShard, indexShard.store()); logger.debug("[{}] closed (reason: [{}])", shardId, reason); } - private void closeShardInjector(String reason, ShardId sId, Injector shardInjector, IndexShard indexShard) { + private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store store) { final int shardId = sId.id(); try { try { indicesLifecycle.beforeIndexShardClosed(sId, indexShard, indexSettings); } finally { - // close everything else even if the beforeIndexShardClosed threw an exception - for (Class closeable : pluginsService.shardServices()) { - try { - shardInjector.getInstance(closeable).close(); - } catch (Throwable e) { - logger.debug("[{}] failed to clean plugin shard service [{}]", e, shardId, closeable); - } - } // this logic is tricky, we want to close the engine so we rollback the changes done to it // and close the shard so no operations are allowed to it if (indexShard != null) { @@ -449,30 +343,13 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } finally { try { - shardInjector.getInstance(Store.class).close(); + store.close(); } catch (Throwable e) { logger.warn("[{}] failed to close store on shard removal (reason: [{}])", e, shardId, reason); } } } - /** - * Closes an optional resource. Returns true if the resource was found; - * NOTE: this method swallows all exceptions thrown from the close method of the injector and logs them as debug log - */ - private boolean closeInjectorOptionalResource(ShardId shardId, Injector shardInjector, Class toClose) { - try { - final Closeable instance = shardInjector.getInstance(toClose); - if (instance == null) { - return false; - } - IOUtils.close(instance); - } catch (Throwable t) { - logger.debug("{} failed to close {}", t, shardId, Strings.toUnderscoreCase(toClose.getSimpleName())); - } - return true; - } - private void onShardClose(ShardLock lock, boolean ownsShard) { if (deleted.get()) { // we remove that shards content if this index has been deleted @@ -492,6 +369,10 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } + public IndexServicesProvider getIndexServices() { + return indexServicesProvider; + } + private class StoreCloseListener implements Store.OnClose { private final ShardId shardId; private final boolean ownsShard; @@ -533,7 +414,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone @Override public void onCache(ShardId shardId, Accountable accountable) { if (shardId != null) { - final IndexShard shard = indexService.shard(shardId.id()); + final IndexShard shard = indexService.getShardOrNull(shardId.id()); if (shard != null) { long ramBytesUsed = accountable != null ? accountable.ramBytesUsed() : 0l; shard.shardBitsetFilterCache().onCached(ramBytesUsed); @@ -544,7 +425,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone @Override public void onRemoval(ShardId shardId, Accountable accountable) { if (shardId != null) { - final IndexShard shard = indexService.shard(shardId.id()); + final IndexShard shard = indexService.getShardOrNull(shardId.id()); if (shard != null) { long ramBytesUsed = accountable != null ? accountable.ramBytesUsed() : 0l; shard.shardBitsetFilterCache().onRemoval(ramBytesUsed); @@ -563,7 +444,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone @Override public void onCache(ShardId shardId, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, Accountable ramUsage) { if (shardId != null) { - final IndexShard shard = indexService.shard(shardId.id()); + final IndexShard shard = indexService.getShardOrNull(shardId.id()); if (shard != null) { shard.fieldData().onCache(shardId, fieldNames, fieldDataType, ramUsage); } @@ -573,7 +454,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone @Override public void onRemoval(ShardId shardId, MappedFieldType.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes) { if (shardId != null) { - final IndexShard shard = indexService.shard(shardId.id()); + final IndexShard shard = indexService.getShardOrNull(shardId.id()); if (shard != null) { shard.fieldData().onRemoval(shardId, fieldNames, fieldDataType, wasEvicted, sizeInBytes); } diff --git a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java new file mode 100644 index 00000000000..fe8428425e2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java @@ -0,0 +1,138 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.aliases.IndexAliasesService; +import org.elasticsearch.index.cache.IndexCache; +import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.shard.IndexSearcherWrapper; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.index.termvectors.TermVectorsService; +import org.elasticsearch.indices.IndicesLifecycle; +import org.elasticsearch.indices.IndicesWarmer; +import org.elasticsearch.indices.cache.query.IndicesQueryCache; +import org.elasticsearch.threadpool.ThreadPool; + +/** + * Simple provider class that holds the Index and Node level services used by + * a shard. + * This is just a temporary solution until we cleaned up index creation and removed injectors on that level as well. + */ +public final class IndexServicesProvider { + + private final IndicesLifecycle indicesLifecycle; + private final ThreadPool threadPool; + private final MapperService mapperService; + private final IndexQueryParserService queryParserService; + private final IndexCache indexCache; + private final IndexAliasesService indexAliasesService; + private final IndicesQueryCache indicesQueryCache; + private final CodecService codecService; + private final TermVectorsService termVectorsService; + private final IndexFieldDataService indexFieldDataService; + private final IndicesWarmer warmer; + private final SimilarityService similarityService; + private final EngineFactory factory; + private final BigArrays bigArrays; + private final IndexSearcherWrapper indexSearcherWrapper; + + @Inject + public IndexServicesProvider(IndicesLifecycle indicesLifecycle, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays, @Nullable IndexSearcherWrapper indexSearcherWrapper) { + this.indicesLifecycle = indicesLifecycle; + this.threadPool = threadPool; + this.mapperService = mapperService; + this.queryParserService = queryParserService; + this.indexCache = indexCache; + this.indexAliasesService = indexAliasesService; + this.indicesQueryCache = indicesQueryCache; + this.codecService = codecService; + this.termVectorsService = termVectorsService; + this.indexFieldDataService = indexFieldDataService; + this.warmer = warmer; + this.similarityService = similarityService; + this.factory = factory; + this.bigArrays = bigArrays; + this.indexSearcherWrapper = indexSearcherWrapper; + } + + public IndicesLifecycle getIndicesLifecycle() { + return indicesLifecycle; + } + + public ThreadPool getThreadPool() { + return threadPool; + } + + public MapperService getMapperService() { + return mapperService; + } + + public IndexQueryParserService getQueryParserService() { + return queryParserService; + } + + public IndexCache getIndexCache() { + return indexCache; + } + + public IndexAliasesService getIndexAliasesService() { + return indexAliasesService; + } + + public IndicesQueryCache getIndicesQueryCache() { + return indicesQueryCache; + } + + public CodecService getCodecService() { + return codecService; + } + + public TermVectorsService getTermVectorsService() { + return termVectorsService; + } + + public IndexFieldDataService getIndexFieldDataService() { + return indexFieldDataService; + } + + public IndicesWarmer getWarmer() { + return warmer; + } + + public SimilarityService getSimilarityService() { + return similarityService; + } + + public EngineFactory getFactory() { + return factory; + } + + public BigArrays getBigArrays() { + return bigArrays; + } + + public IndexSearcherWrapper getIndexSearcherWrapper() { return indexSearcherWrapper; } +} diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index f9331c4416a..1330ef05a7f 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -59,6 +59,8 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; +import java.util.function.Supplier; /** * @@ -78,7 +80,6 @@ public abstract class Engine implements Closeable { protected final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); protected final ReleasableLock readLock = new ReleasableLock(rwl.readLock()); protected final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock()); - protected volatile Throwable failedEngine = null; protected Engine(EngineConfig engineConfig) { @@ -227,8 +228,8 @@ public abstract class Engine implements Closeable { PENDING_OPERATIONS } - final protected GetResult getFromSearcher(Get get) throws EngineException { - final Searcher searcher = acquireSearcher("get"); + final protected GetResult getFromSearcher(Get get, Function searcherFactory) throws EngineException { + final Searcher searcher = searcherFactory.apply("get"); final Versions.DocIdAndVersion docIdAndVersion; try { docIdAndVersion = Versions.loadDocIdAndVersion(searcher.reader(), get.uid()); @@ -256,7 +257,11 @@ public abstract class Engine implements Closeable { } } - public abstract GetResult get(Get get) throws EngineException; + public final GetResult get(Get get) throws EngineException { + return get(get, this::acquireSearcher); + } + + public abstract GetResult get(Get get, Function searcherFactory) throws EngineException; /** * Returns a new searcher instance. The consumer of this @@ -279,7 +284,7 @@ public abstract class Engine implements Closeable { try { final Searcher retVal = newSearcher(source, searcher, manager); success = true; - return config().getWrappingService().wrap(engineConfig, retVal); + return retVal; } finally { if (!success) { manager.release(searcher); diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index c6e67243514..a79587e4347 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.SnapshotDeletionPolicy; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -32,6 +33,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.indexing.ShardIndexingService; +import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.TranslogRecoveryPerformer; @@ -73,7 +75,7 @@ public final class EngineConfig { private final boolean forceNewTranslog; private final QueryCache queryCache; private final QueryCachingPolicy queryCachingPolicy; - private final IndexSearcherWrappingService wrappingService; + private final SetOnce searcherWrapper = new SetOnce<>(); /** * Index setting for compound file on flush. This setting is realtime updateable. @@ -121,7 +123,7 @@ public final class EngineConfig { Settings indexSettings, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy, MergePolicy mergePolicy, MergeSchedulerConfig mergeSchedulerConfig, Analyzer analyzer, Similarity similarity, CodecService codecService, Engine.FailedEngineListener failedEngineListener, - TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, IndexSearcherWrappingService wrappingService, TranslogConfig translogConfig) { + TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, TranslogConfig translogConfig) { this.shardId = shardId; this.indexSettings = indexSettings; this.threadPool = threadPool; @@ -135,7 +137,6 @@ public final class EngineConfig { this.similarity = similarity; this.codecService = codecService; this.failedEngineListener = failedEngineListener; - this.wrappingService = wrappingService; this.compoundOnFlush = indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush); codecName = indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME); indexingBufferSize = DEFAULT_INDEX_BUFFER_SIZE; @@ -380,10 +381,6 @@ public final class EngineConfig { return queryCachingPolicy; } - public IndexSearcherWrappingService getWrappingService() { - return wrappingService; - } - /** * Returns the translog config for this engine */ diff --git a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java deleted file mode 100644 index 665d17a2f86..00000000000 --- a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.engine; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.search.IndexSearcher; - -/** - * Extension point to add custom functionality at request time to the {@link DirectoryReader} - * and {@link IndexSearcher} managed by the {@link Engine}. - */ -public interface IndexSearcherWrapper { - - /** - * @param reader The provided directory reader to be wrapped to add custom functionality - * @return a new directory reader wrapping the provided directory reader or if no wrapping was performed - * the provided directory reader - */ - DirectoryReader wrap(DirectoryReader reader); - - /** - * @param engineConfig The engine config which can be used to get the query cache and query cache policy from - * when creating a new index searcher - * @param searcher The provided index searcher to be wrapped to add custom functionality - * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed - * the provided index searcher - */ - IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException; - -} diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 5b76040da5e..227212dd86e 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -66,6 +66,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.function.Supplier; /** * @@ -303,7 +305,7 @@ public class InternalEngine extends Engine { } @Override - public GetResult get(Get get) throws EngineException { + public GetResult get(Get get, Function searcherFactory) throws EngineException { try (ReleasableLock lock = readLock.acquire()) { ensureOpen(); if (get.realtime()) { @@ -324,7 +326,7 @@ public class InternalEngine extends Engine { } // no version, get the version from the index, we know that we refresh on flush - return getFromSearcher(get); + return getFromSearcher(get, searcherFactory); } } diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java index f589b289c17..7588ffae355 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.translog.Translog; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.function.Function; /** * ShadowEngine is a specialized engine that only allows read-only operations @@ -168,9 +169,9 @@ public class ShadowEngine extends Engine { } @Override - public GetResult get(Get get) throws EngineException { + public GetResult get(Get get, Function searcherFacotry) throws EngineException { // There is no translog, so we can get it directly from the searcher - return getFromSearcher(get); + return getFromSearcher(get, searcherFacotry); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/percolator/stats/PercolateStats.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java similarity index 99% rename from core/src/main/java/org/elasticsearch/index/percolator/stats/PercolateStats.java rename to core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java index 49f2375a03a..f927a42761f 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/stats/PercolateStats.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.percolator.stats; +package org.elasticsearch.index.percolator; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java index 7dd26ec55db..d811f1f6e71 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.percolator; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -27,6 +28,8 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.metrics.CounterMetric; +import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -41,20 +44,18 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentTypeListener; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; -import org.elasticsearch.index.percolator.stats.ShardPercolateService; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; -import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.percolator.PercolatorService; import java.io.Closeable; import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -64,39 +65,35 @@ import java.util.concurrent.atomic.AtomicBoolean; * Once a document type has been created, the real-time percolator will start to listen to write events and update the * this registry with queries in real time. */ -public class PercolatorQueriesRegistry extends AbstractIndexShardComponent implements Closeable{ +public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent implements Closeable { public final String MAP_UNMAPPED_FIELDS_AS_STRING = "index.percolator.map_unmapped_fields_as_string"; // This is a shard level service, but these below are index level service: private final IndexQueryParserService queryParserService; private final MapperService mapperService; - private final IndicesLifecycle indicesLifecycle; private final IndexFieldDataService indexFieldDataService; private final ShardIndexingService indexingService; - private final ShardPercolateService shardPercolateService; private final ConcurrentMap percolateQueries = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(); - private final ShardLifecycleListener shardLifecycleListener = new ShardLifecycleListener(); private final RealTimePercolatorOperationListener realTimePercolatorOperationListener = new RealTimePercolatorOperationListener(); private final PercolateTypeListener percolateTypeListener = new PercolateTypeListener(); private final AtomicBoolean realTimePercolatorEnabled = new AtomicBoolean(false); private boolean mapUnmappedFieldsAsString; + private final MeanMetric percolateMetric = new MeanMetric(); + private final CounterMetric currentMetric = new CounterMetric(); + private final CounterMetric numberOfQueries = new CounterMetric(); public PercolatorQueriesRegistry(ShardId shardId, @IndexSettings Settings indexSettings, IndexQueryParserService queryParserService, - ShardIndexingService indexingService, IndicesLifecycle indicesLifecycle, MapperService mapperService, - IndexFieldDataService indexFieldDataService, ShardPercolateService shardPercolateService) { + ShardIndexingService indexingService, MapperService mapperService, + IndexFieldDataService indexFieldDataService) { super(shardId, indexSettings); this.queryParserService = queryParserService; this.mapperService = mapperService; - this.indicesLifecycle = indicesLifecycle; this.indexingService = indexingService; this.indexFieldDataService = indexFieldDataService; - this.shardPercolateService = shardPercolateService; this.mapUnmappedFieldsAsString = indexSettings.getAsBoolean(MAP_UNMAPPED_FIELDS_AS_STRING, false); - - indicesLifecycle.addListener(shardLifecycleListener); mapperService.addTypeListener(percolateTypeListener); } @@ -107,7 +104,6 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple @Override public void close() { mapperService.removeTypeListener(percolateTypeListener); - indicesLifecycle.removeListener(shardLifecycleListener); indexingService.removeListener(realTimePercolatorOperationListener); clear(); } @@ -116,30 +112,25 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple percolateQueries.clear(); } - void enableRealTimePercolator() { + public void enableRealTimePercolator() { if (realTimePercolatorEnabled.compareAndSet(false, true)) { indexingService.addListener(realTimePercolatorOperationListener); } } - void disableRealTimePercolator() { - if (realTimePercolatorEnabled.compareAndSet(true, false)) { - indexingService.removeListener(realTimePercolatorOperationListener); - } - } - public void addPercolateQuery(String idAsString, BytesReference source) { Query newquery = parsePercolatorDocument(idAsString, source); BytesRef id = new BytesRef(idAsString); - Query previousQuery = percolateQueries.put(id, newquery); - shardPercolateService.addedQuery(id, previousQuery, newquery); + percolateQueries.put(id, newquery); + numberOfQueries.inc(); + } public void removePercolateQuery(String idAsString) { BytesRef id = new BytesRef(idAsString); Query query = percolateQueries.remove(id); if (query != null) { - shardPercolateService.removedQuery(id, query); + numberOfQueries.dec(); } } @@ -225,55 +216,27 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple enableRealTimePercolator(); } } - } - private class ShardLifecycleListener extends IndicesLifecycle.Listener { - - @Override - public void afterIndexShardCreated(IndexShard indexShard) { - if (hasPercolatorType(indexShard)) { - enableRealTimePercolator(); + public void loadQueries(IndexReader reader) { + logger.trace("loading percolator queries..."); + final int loadedQueries; + try { + Query query = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME)); + QueriesLoaderCollector queryCollector = new QueriesLoaderCollector(PercolatorQueriesRegistry.this, logger, mapperService, indexFieldDataService); + IndexSearcher indexSearcher = new IndexSearcher(reader); + indexSearcher.setQueryCache(null); + indexSearcher.search(query, queryCollector); + Map queries = queryCollector.queries(); + for (Map.Entry entry : queries.entrySet()) { + percolateQueries.put(entry.getKey(), entry.getValue()); + numberOfQueries.inc(); } + loadedQueries = queries.size(); + } catch (Exception e) { + throw new PercolatorException(shardId.index(), "failed to load queries from percolator index", e); } - - @Override - public void beforeIndexShardPostRecovery(IndexShard indexShard) { - if (hasPercolatorType(indexShard)) { - // percolator index has started, fetch what we can from it and initialize the indices - // we have - logger.trace("loading percolator queries for [{}]...", shardId); - int loadedQueries = loadQueries(indexShard); - logger.debug("done loading [{}] percolator queries for [{}]", loadedQueries, shardId); - } - } - - private boolean hasPercolatorType(IndexShard indexShard) { - ShardId otherShardId = indexShard.shardId(); - return shardId.equals(otherShardId) && mapperService.hasMapping(PercolatorService.TYPE_NAME); - } - - private int loadQueries(IndexShard shard) { - shard.refresh("percolator_load_queries"); - // NOTE: we acquire the searcher via the engine directly here since this is executed right - // before the shard is marked as POST_RECOVERY - try (Engine.Searcher searcher = shard.engine().acquireSearcher("percolator_load_queries")) { - Query query = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME)); - QueriesLoaderCollector queryCollector = new QueriesLoaderCollector(PercolatorQueriesRegistry.this, logger, mapperService, indexFieldDataService); - IndexSearcher indexSearcher = new IndexSearcher(searcher.reader()); - indexSearcher.setQueryCache(null); - indexSearcher.search(query, queryCollector); - Map queries = queryCollector.queries(); - for (Map.Entry entry : queries.entrySet()) { - Query previousQuery = percolateQueries.put(entry.getKey(), entry.getValue()); - shardPercolateService.addedQuery(entry.getKey(), previousQuery, entry.getValue()); - } - return queries.size(); - } catch (Exception e) { - throw new PercolatorException(shardId.index(), "failed to load queries from percolator index", e); - } - } - + logger.debug("done loading [{}] percolator queries", loadedQueries); } private class RealTimePercolatorOperationListener extends IndexingOperationListener { @@ -320,4 +283,35 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple } } } + + public void prePercolate() { + currentMetric.inc(); + } + + public void postPercolate(long tookInNanos) { + currentMetric.dec(); + percolateMetric.inc(tookInNanos); + } + + /** + * @return The current metrics + */ + public PercolateStats stats() { + return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), -1, numberOfQueries.count()); + } + + // Enable when a more efficient manner is found for estimating the size of a Lucene query. + /*private static long computeSizeInMemory(HashedBytesRef id, Query query) { + long size = (3 * RamUsageEstimator.NUM_BYTES_INT) + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + id.bytes.bytes.length; + size += RamEstimator.sizeOf(query); + return size; + } + + private static final class RamEstimator { + // we move this into it's own class to exclude it from the forbidden API checks + // it's fine to use here! + static long sizeOf(Query query) { + return RamUsageEstimator.sizeOf(query); + } + }*/ } diff --git a/core/src/main/java/org/elasticsearch/index/percolator/stats/ShardPercolateService.java b/core/src/main/java/org/elasticsearch/index/percolator/stats/ShardPercolateService.java deleted file mode 100644 index 80f6bd9be38..00000000000 --- a/core/src/main/java/org/elasticsearch/index/percolator/stats/ShardPercolateService.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.percolator.stats; - -import org.apache.lucene.search.Query; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.metrics.MeanMetric; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.AbstractIndexShardComponent; -import org.elasticsearch.index.shard.ShardId; - -import java.util.concurrent.TimeUnit; - -/** - * Shard level percolator service that maintains percolator metrics: - *
    - *
  • total time spent in percolate api - *
  • the current number of percolate requests - *
  • number of registered percolate queries - *
- */ -public class ShardPercolateService extends AbstractIndexShardComponent { - - @Inject - public ShardPercolateService(ShardId shardId, @IndexSettings Settings indexSettings) { - super(shardId, indexSettings); - } - - private final MeanMetric percolateMetric = new MeanMetric(); - private final CounterMetric currentMetric = new CounterMetric(); - - private final CounterMetric numberOfQueries = new CounterMetric(); - - public void prePercolate() { - currentMetric.inc(); - } - - public void postPercolate(long tookInNanos) { - currentMetric.dec(); - percolateMetric.inc(tookInNanos); - } - - public void addedQuery(BytesRef id, Query previousQuery, Query newQuery) { - numberOfQueries.inc(); - } - - public void removedQuery(BytesRef id, Query query) { - numberOfQueries.dec(); - } - - /** - * @return The current metrics - */ - public PercolateStats stats() { - return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), -1, numberOfQueries.count()); - } - - // Enable when a more efficient manner is found for estimating the size of a Lucene query. - /*private static long computeSizeInMemory(HashedBytesRef id, Query query) { - long size = (3 * RamUsageEstimator.NUM_BYTES_INT) + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + id.bytes.bytes.length; - size += RamEstimator.sizeOf(query); - return size; - } - - private static final class RamEstimator { - // we move this into it's own class to exclude it from the forbidden API checks - // it's fine to use here! - static long sizeOf(Query query) { - return RamUsageEstimator.sizeOf(query); - } - }*/ - -} diff --git a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrappingService.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java similarity index 50% rename from core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrappingService.java rename to core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java index 23d05f01dc7..c75f3c7995f 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrappingService.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -17,59 +17,47 @@ * under the License. */ -package org.elasticsearch.index.engine; +package org.elasticsearch.index.shard; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.index.engine.Engine.Searcher; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineException; -import java.util.Set; +import java.io.IOException; /** - * Service responsible for wrapping the {@link DirectoryReader} and {@link IndexSearcher} of a {@link Searcher} via the - * configured {@link IndexSearcherWrapper} instance. This allows custom functionally to be added the {@link Searcher} - * before being used to do an operation (search, get, field stats etc.) + * Extension point to add custom functionality at request time to the {@link DirectoryReader} + * and {@link IndexSearcher} managed by the {@link Engine}. */ -// TODO: This needs extension point is a bit hacky now, because the IndexSearch from the engine can only be wrapped once, -// if we allowed the IndexSearcher to be wrapped multiple times then a custom IndexSearcherWrapper needs have good -// control over its location in the wrapping chain -public final class IndexSearcherWrappingService { +public interface IndexSearcherWrapper { - private final IndexSearcherWrapper wrapper; + /** + * @param reader The provided directory reader to be wrapped to add custom functionality + * @return a new directory reader wrapping the provided directory reader or if no wrapping was performed + * the provided directory reader + */ + DirectoryReader wrap(DirectoryReader reader) throws IOException; - // for unit tests: - IndexSearcherWrappingService() { - this.wrapper = null; - } - - @Inject - // Use a Set parameter here, because constructor parameter can't be optional - // and I prefer to keep the `wrapper` field final. - public IndexSearcherWrappingService(Set wrappers) { - if (wrappers.size() > 1) { - throw new IllegalStateException("wrapping of the index searcher by more than one wrappers is forbidden, found the following wrappers [" + wrappers + "]"); - } - if (wrappers.isEmpty()) { - this.wrapper = null; - } else { - this.wrapper = wrappers.iterator().next(); - } - } + /** + * @param engineConfig The engine config which can be used to get the query cache and query cache policy from + * when creating a new index searcher + * @param searcher The provided index searcher to be wrapped to add custom functionality + * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed + * the provided index searcher + */ + IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws IOException; /** * If there are configured {@link IndexSearcherWrapper} instances, the {@link IndexSearcher} of the provided engine searcher - * gets wrapped and a new {@link Searcher} instances is returned, otherwise the provided {@link Searcher} is returned. + * gets wrapped and a new {@link Engine.Searcher} instances is returned, otherwise the provided {@link Engine.Searcher} is returned. * - * This is invoked each time a {@link Searcher} is requested to do an operation. (for example search) + * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search) */ - public Searcher wrap(EngineConfig engineConfig, final Searcher engineSearcher) throws EngineException { - if (wrapper == null) { - return engineSearcher; - } - - DirectoryReader reader = wrapper.wrap((DirectoryReader) engineSearcher.reader()); + default Engine.Searcher wrap(EngineConfig engineConfig, Engine.Searcher engineSearcher) throws IOException { + DirectoryReader reader = wrap((DirectoryReader) engineSearcher.reader()); IndexSearcher innerIndexSearcher = new IndexSearcher(reader); innerIndexSearcher.setQueryCache(engineConfig.getQueryCache()); innerIndexSearcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy()); @@ -77,12 +65,11 @@ public final class IndexSearcherWrappingService { // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times - IndexSearcher indexSearcher = wrapper.wrap(engineConfig, innerIndexSearcher); + IndexSearcher indexSearcher = wrap(engineConfig, innerIndexSearcher); if (reader == engineSearcher.reader() && indexSearcher == innerIndexSearcher) { return engineSearcher; } else { return new Engine.Searcher(engineSearcher.source(), indexSearcher) { - @Override public void close() throws ElasticsearchException { engineSearcher.close(); diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index c98a9c0f9dd..ea2d555ae0d 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -20,10 +20,7 @@ package org.elasticsearch.index.shard; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.index.CheckIndex; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; -import org.apache.lucene.index.SnapshotDeletionPolicy; +import org.apache.lucene.index.*; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.store.AlreadyClosedException; @@ -36,6 +33,7 @@ import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; +import org.elasticsearch.bootstrap.Elasticsearch; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -51,11 +49,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AbstractRefCounted; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.gateway.MetaDataStateFormat; +import org.elasticsearch.index.IndexServicesProvider; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.aliases.IndexAliasesService; import org.elasticsearch.index.cache.IndexCache; @@ -75,8 +73,8 @@ import org.elasticsearch.index.indexing.IndexingStats; import org.elasticsearch.index.indexing.ShardIndexingService; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.percolator.PercolateStats; import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; -import org.elasticsearch.index.percolator.stats.ShardPercolateService; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; @@ -99,12 +97,12 @@ import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.translog.TranslogWriter; import org.elasticsearch.index.warmer.ShardIndexWarmerService; import org.elasticsearch.index.warmer.WarmerStats; -import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.InternalIndicesLifecycle; import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.recovery.RecoveryFailedException; import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.percolator.PercolatorService; import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.threadpool.ThreadPool; @@ -137,7 +135,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett private final ShardRequestCache shardQueryCache; private final ShardFieldData shardFieldData; private final PercolatorQueriesRegistry percolatorQueriesRegistry; - private final ShardPercolateService shardPercolateService; private final TermVectorsService termVectorsService; private final IndexFieldDataService indexFieldDataService; private final ShardSuggestMetric shardSuggestMetric = new ShardSuggestMetric(); @@ -161,7 +158,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett protected volatile IndexShardState state; protected final AtomicReference currentEngineReference = new AtomicReference<>(); protected final EngineFactory engineFactory; - private final IndexSearcherWrappingService wrappingService; @Nullable private RecoveryState recoveryState; @@ -190,42 +186,36 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett private final IndexShardOperationCounter indexShardOperationCounter; - private EnumSet readAllowedStates = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY); + private final EnumSet readAllowedStates = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY); + + private final IndexSearcherWrapper searcherWrapper; @Inject - public IndexShard(ShardId shardId, @IndexSettings Settings indexSettings, IndicesLifecycle indicesLifecycle, Store store, - ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, - IndicesQueryCache indicesQueryCache, CodecService codecService, - TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, - @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, - ShardPath path, BigArrays bigArrays, IndexSearcherWrappingService wrappingService) { + public IndexShard(ShardId shardId, @IndexSettings Settings indexSettings, ShardPath path, Store store, IndexServicesProvider provider) { super(shardId, indexSettings); - this.codecService = codecService; - this.warmer = warmer; + this.codecService = provider.getCodecService(); + this.warmer = provider.getWarmer(); this.deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - this.similarityService = similarityService; - this.wrappingService = wrappingService; + this.similarityService = provider.getSimilarityService(); Objects.requireNonNull(store, "Store must be provided to the index shard"); - this.engineFactory = factory; - this.indicesLifecycle = (InternalIndicesLifecycle) indicesLifecycle; + this.engineFactory = provider.getFactory(); + this.indicesLifecycle = (InternalIndicesLifecycle) provider.getIndicesLifecycle(); this.store = store; this.mergeSchedulerConfig = new MergeSchedulerConfig(indexSettings); - this.threadPool = threadPool; - this.mapperService = mapperService; - this.queryParserService = queryParserService; - this.indexCache = indexCache; - this.indexAliasesService = indexAliasesService; + this.threadPool = provider.getThreadPool(); + this.mapperService = provider.getMapperService(); + this.queryParserService = provider.getQueryParserService(); + this.indexCache = provider.getIndexCache(); + this.indexAliasesService = provider.getIndexAliasesService(); this.indexingService = new ShardIndexingService(shardId, indexSettings); this.getService = new ShardGetService(this, mapperService); - this.termVectorsService = termVectorsService; + this.termVectorsService = provider.getTermVectorsService(); this.searchService = new ShardSearchStats(indexSettings); this.shardWarmerService = new ShardIndexWarmerService(shardId, indexSettings); - this.indicesQueryCache = indicesQueryCache; + this.indicesQueryCache = provider.getIndicesQueryCache(); this.shardQueryCache = new ShardRequestCache(shardId, indexSettings); this.shardFieldData = new ShardFieldData(); - this.shardPercolateService = new ShardPercolateService(shardId, indexSettings); - this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryParserService, indexingService, indicesLifecycle, mapperService, indexFieldDataService, shardPercolateService); - this.indexFieldDataService = indexFieldDataService; + this.indexFieldDataService = provider.getIndexFieldDataService(); this.shardBitsetFilterCache = new ShardBitsetFilterCache(shardId, indexSettings); state = IndexShardState.CREATED; this.refreshInterval = indexSettings.getAsTime(INDEX_REFRESH_INTERVAL, EngineConfig.DEFAULT_REFRESH_INTERVAL); @@ -238,7 +228,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett this.checkIndexOnStartup = indexSettings.get("index.shard.check_on_startup", "false"); this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, getFromSettings(logger, indexSettings, Translog.Durabilty.REQUEST), - bigArrays, threadPool); + provider.getBigArrays(), threadPool); final QueryCachingPolicy cachingPolicy; // the query cache is a node-level thing, however we want the most popular filters // to be computed on a per-shard basis @@ -252,6 +242,11 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett this.flushThresholdSize = indexSettings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB)); this.disableFlush = indexSettings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false); this.indexShardOperationCounter = new IndexShardOperationCounter(logger, shardId); + this.searcherWrapper = provider.getIndexSearcherWrapper(); + this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryParserService, indexingService, mapperService, indexFieldDataService); + if (mapperService.hasMapping(PercolatorService.TYPE_NAME)) { + percolatorQueriesRegistry.enableRealTimePercolator(); + } } public Store store() { @@ -344,7 +339,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett if (newRouting.state() == ShardRoutingState.STARTED || newRouting.state() == ShardRoutingState.RELOCATING) { // we want to refresh *before* we move to internal STARTED state try { - engine().refresh("cluster_state_started"); + getEngine().refresh("cluster_state_started"); } catch (Throwable t) { logger.debug("failed to refresh due to move to cluster wide started", t); } @@ -453,7 +448,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett if (logger.isTraceEnabled()) { logger.trace("index [{}][{}]{}", create.type(), create.id(), create.docs()); } - engine().create(create); + getEngine().create(create); create.endTime(System.nanoTime()); } catch (Throwable ex) { indexingService.postCreate(create, ex); @@ -492,7 +487,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett if (logger.isTraceEnabled()) { logger.trace("index [{}][{}]{}", index.type(), index.id(), index.docs()); } - created = engine().index(index); + created = getEngine().index(index); index.endTime(System.nanoTime()); } catch (Throwable ex) { indexingService.postIndex(index, ex); @@ -515,7 +510,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett if (logger.isTraceEnabled()) { logger.trace("delete [{}]", delete.uid().text()); } - engine().delete(delete); + getEngine().delete(delete); delete.endTime(System.nanoTime()); } catch (Throwable ex) { indexingService.postDelete(delete, ex); @@ -526,7 +521,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett public Engine.GetResult get(Engine.Get get) { readAllowed(); - return engine().get(get); + return getEngine().get(get, this::acquireSearcher); } public void refresh(String source) { @@ -535,7 +530,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett logger.trace("refresh with source: {}", source); } long time = System.nanoTime(); - engine().refresh(source); + getEngine().refresh(source); refreshMetric.inc(System.nanoTime() - time); } @@ -561,7 +556,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett */ @Nullable public CommitStats commitStats() { - Engine engine = engineUnsafe(); + Engine engine = getEngineOrNull(); return engine == null ? null : engine.commitStats(); } @@ -588,7 +583,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } public MergeStats mergeStats() { - final Engine engine = engineUnsafe(); + final Engine engine = getEngineOrNull(); if (engine == null) { return new MergeStats(); } @@ -596,7 +591,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } public SegmentsStats segmentStats() { - SegmentsStats segmentsStats = engine().segmentsStats(); + SegmentsStats segmentsStats = getEngine().segmentsStats(); segmentsStats.addBitsetMemoryInBytes(shardBitsetFilterCache.getMemorySizeInBytes()); return segmentsStats; } @@ -621,12 +616,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett return percolatorQueriesRegistry; } - public ShardPercolateService shardPercolateService() { - return shardPercolateService; - } - public TranslogStats translogStats() { - return engine().getTranslog().stats(); + return getEngine().getTranslog().stats(); } public SuggestStats suggestStats() { @@ -651,7 +642,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett public Engine.SyncedFlushResult syncFlush(String syncId, Engine.CommitId expectedCommitId) { verifyStartedOrRecovering(); logger.trace("trying to sync flush. sync id [{}]. expected commit id [{}]]", syncId, expectedCommitId); - return engine().syncFlush(syncId, expectedCommitId); + return getEngine().syncFlush(syncId, expectedCommitId); } public Engine.CommitId flush(FlushRequest request) throws ElasticsearchException { @@ -666,7 +657,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett verifyStartedOrRecovering(); long time = System.nanoTime(); - Engine.CommitId commitId = engine().flush(force, waitIfOngoing); + Engine.CommitId commitId = getEngine().flush(force, waitIfOngoing); flushMetric.inc(System.nanoTime() - time); return commitId; @@ -677,7 +668,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett if (logger.isTraceEnabled()) { logger.trace("optimize with {}", optimize); } - engine().forceMerge(optimize.flush(), optimize.maxNumSegments(), optimize.onlyExpungeDeletes(), false, false); + getEngine().forceMerge(optimize.flush(), optimize.maxNumSegments(), optimize.onlyExpungeDeletes(), false, false); } /** @@ -690,7 +681,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } org.apache.lucene.util.Version previousVersion = minimumCompatibleVersion(); // we just want to upgrade the segments, not actually optimize to a single segment - engine().forceMerge(true, // we need to flush at the end to make sure the upgrade is durable + getEngine().forceMerge(true, // we need to flush at the end to make sure the upgrade is durable Integer.MAX_VALUE, // we just want to upgrade the segments, not actually optimize to a single segment false, true, upgrade.upgradeOnlyAncientSegments()); org.apache.lucene.util.Version version = minimumCompatibleVersion(); @@ -703,7 +694,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett public org.apache.lucene.util.Version minimumCompatibleVersion() { org.apache.lucene.util.Version luceneVersion = null; - for (Segment segment : engine().segments(false)) { + for (Segment segment : getEngine().segments(false)) { if (luceneVersion == null || luceneVersion.onOrAfter(segment.getVersion())) { luceneVersion = segment.getVersion(); } @@ -721,7 +712,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett IndexShardState state = this.state; // one time volatile read // we allow snapshot on closed index shard, since we want to do one after we close the shard and before we close the engine if (state == IndexShardState.STARTED || state == IndexShardState.RELOCATED || state == IndexShardState.CLOSED) { - return engine().snapshotIndex(flushFirst); + return getEngine().snapshotIndex(flushFirst); } else { throw new IllegalIndexShardStateException(shardId, state, "snapshot is not allowed"); } @@ -742,12 +733,17 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett */ public void failShard(String reason, @Nullable Throwable e) { // fail the engine. This will cause this shard to also be removed from the node's index service. - engine().failEngine(reason, e); + getEngine().failEngine(reason, e); } public Engine.Searcher acquireSearcher(String source) { readAllowed(); - return engine().acquireSearcher(source); + Engine engine = getEngine(); + try { + return searcherWrapper == null ? engine.acquireSearcher(source) : searcherWrapper.wrap(engineConfig, engine.acquireSearcher(source)); + } catch (IOException ex) { + throw new ElasticsearchException("failed to wrap searcher", ex); + } } public void close(String reason, boolean flushEngine) throws IOException { @@ -774,8 +770,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } } + public IndexShard postRecovery(String reason) throws IndexShardStartedException, IndexShardRelocatedException, IndexShardClosedException { - indicesLifecycle.beforeIndexShardPostRecovery(this); + if (mapperService.hasMapping(PercolatorService.TYPE_NAME)) { + refresh("percolator_load_queries"); + try (Engine.Searcher searcher = getEngine().acquireSearcher("percolator_load_queries")) { + this.percolatorQueriesRegistry.loadQueries(searcher.reader()); + } + } synchronized (mutex) { if (state == IndexShardState.CLOSED) { throw new IndexShardClosedException(shardId); @@ -789,7 +791,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett recoveryState.setStage(RecoveryState.Stage.DONE); changeState(IndexShardState.POST_RECOVERY, reason); } - indicesLifecycle.afterIndexShardPostRecovery(this); return this; } @@ -813,7 +814,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett if (state != IndexShardState.RECOVERING) { throw new IndexShardNotRecoveringException(shardId, state); } - return engineConfig.getTranslogRecoveryPerformer().performBatchRecovery(engine(), operations); + return engineConfig.getTranslogRecoveryPerformer().performBatchRecovery(getEngine(), operations); } /** @@ -852,7 +853,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett * a remote peer. */ public void skipTranslogRecovery() throws IOException { - assert engineUnsafe() == null : "engine was already created"; + assert getEngineOrNull() == null : "engine was already created"; internalPerformTranslogRecovery(true, true); assert recoveryState.getTranslog().recoveredOperations() == 0; } @@ -892,7 +893,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett */ public void finalizeRecovery() { recoveryState().setStage(RecoveryState.Stage.FINALIZE); - engine().refresh("recovery_finalization"); + getEngine().refresh("recovery_finalization"); startScheduledTasksIfNeeded(); engineConfig.setEnableGcDeletes(true); } @@ -982,7 +983,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett config.setIndexingBufferSize(shardIndexingBufferSize); - Engine engine = engineUnsafe(); + Engine engine = getEngineOrNull(); if (engine == null) { logger.debug("updateBufferSize: engine is closed; skipping"); return; @@ -1057,7 +1058,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett */ boolean shouldFlush() { if (disableFlush == false) { - Engine engine = engineUnsafe(); + Engine engine = getEngineOrNull(); if (engine != null) { try { Translog translog = engine.getTranslog(); @@ -1171,15 +1172,37 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett searchService.onRefreshSettings(settings); indexingService.onRefreshSettings(settings); if (change) { - engine().onSettingsChanged(); + getEngine().onSettingsChanged(); } } + public Translog.View acquireTranslogView() { + Engine engine = getEngine(); + assert engine.getTranslog() != null : "translog must not be null"; + return engine.getTranslog().newView(); + } + + public List segments(boolean verbose) { + return getEngine().segments(verbose); + } + + public void flushAndCloseEngine() throws IOException { + getEngine().flushAndClose(); + } + + public Translog getTranslog() { + return getEngine().getTranslog(); + } + + public PercolateStats percolateStats() { + return percolatorQueriesRegistry.stats(); + } + class EngineRefresher implements Runnable { @Override public void run() { // we check before if a refresh is needed, if not, we reschedule, otherwise, we fork, refresh, and then reschedule - if (!engine().refreshNeeded()) { + if (!getEngine().refreshNeeded()) { reschedule(); return; } @@ -1187,7 +1210,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett @Override public void run() { try { - if (engine().refreshNeeded()) { + if (getEngine().refreshNeeded()) { refresh("schedule"); } } catch (EngineClosedException e) { @@ -1300,8 +1323,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett recoveryState.getVerifyIndex().checkIndexTime(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - timeNS))); } - public Engine engine() { - Engine engine = engineUnsafe(); + Engine getEngine() { + Engine engine = getEngineOrNull(); if (engine == null) { throw new EngineClosedException(shardId); } @@ -1310,7 +1333,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett /** NOTE: returns null if engine is not yet started (e.g. recovery phase 1, copying over index files, is still running), or if engine is * closed. */ - protected Engine engineUnsafe() { + protected Engine getEngineOrNull() { return this.currentEngineReference.get(); } @@ -1403,7 +1426,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett }; return new EngineConfig(shardId, threadPool, indexingService, indexSettings, warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig, - mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, wrappingService, translogConfig); + mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig); } private static class IndexShardOperationCounter extends AbstractRefCounted { @@ -1444,7 +1467,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett */ public void sync(Translog.Location location) { try { - final Engine engine = engine(); + final Engine engine = getEngine(); engine.getTranslog().ensureSynced(location); } catch (EngineClosedException ex) { // that's fine since we already synced everything on engine close - this also is conform with the methods documentation @@ -1515,4 +1538,5 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } return false; } + } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java deleted file mode 100644 index 188669f3fb2..00000000000 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShardModule.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.shard; - -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.multibindings.Multibinder; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.engine.IndexSearcherWrapper; -import org.elasticsearch.index.engine.IndexSearcherWrappingService; -import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.engine.InternalEngineFactory; - -/** - * The {@code IndexShardModule} module is responsible for binding the correct - * shard id, index shard, engine factory, and warming service for a newly - * created shard. - */ -public class IndexShardModule extends AbstractModule { - - private final ShardId shardId; - private final Settings settings; - private final boolean primary; - - // pkg private so tests can mock - Class engineFactoryImpl = InternalEngineFactory.class; - - public IndexShardModule(ShardId shardId, boolean primary, Settings settings) { - this.settings = settings; - this.shardId = shardId; - this.primary = primary; - if (settings.get("index.translog.type") != null) { - throw new IllegalStateException("a custom translog type is no longer supported. got [" + settings.get("index.translog.type") + "]"); - } - } - - /** Return true if a shadow engine should be used */ - protected boolean useShadowEngine() { - return primary == false && IndexMetaData.isIndexUsingShadowReplicas(settings); - } - - @Override - protected void configure() { - bind(ShardId.class).toInstance(shardId); - if (useShadowEngine()) { - bind(IndexShard.class).to(ShadowIndexShard.class).asEagerSingleton(); - } else { - bind(IndexShard.class).asEagerSingleton(); - } - - bind(EngineFactory.class).to(engineFactoryImpl); - bind(IndexSearcherWrappingService.class).asEagerSingleton(); - // this injects an empty set in IndexSearcherWrappingService, otherwise guice can't construct IndexSearcherWrappingService - Multibinder multibinder - = Multibinder.newSetBinder(binder(), IndexSearcherWrapper.class); - } - - -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java index 62fa928faf1..c81b9e5c541 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java @@ -18,32 +18,14 @@ */ package org.elasticsearch.index.shard; -import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.aliases.IndexAliasesService; -import org.elasticsearch.index.cache.IndexCache; -import org.elasticsearch.index.codec.CodecService; -import org.elasticsearch.index.engine.IndexSearcherWrappingService; +import org.elasticsearch.index.IndexServicesProvider; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineConfig; -import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.fielddata.IndexFieldDataService; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.settings.IndexSettingsService; -import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.termvectors.TermVectorsService; -import org.elasticsearch.indices.IndicesLifecycle; -import org.elasticsearch.indices.IndicesWarmer; -import org.elasticsearch.indices.cache.query.IndicesQueryCache; -import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -55,23 +37,8 @@ import java.io.IOException; */ public final class ShadowIndexShard extends IndexShard { - @Inject - public ShadowIndexShard(ShardId shardId, @IndexSettings Settings indexSettings, - IndicesLifecycle indicesLifecycle, Store store, - ThreadPool threadPool, MapperService mapperService, - IndexQueryParserService queryParserService, IndexCache indexCache, - IndexAliasesService indexAliasesService, IndicesQueryCache indicesQueryCache, - CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, - @Nullable IndicesWarmer warmer, - SimilarityService similarityService, - EngineFactory factory, - ShardPath path, BigArrays bigArrays, IndexSearcherWrappingService wrappingService) throws IOException { - super(shardId, indexSettings, indicesLifecycle, store, - threadPool, mapperService, queryParserService, indexCache, indexAliasesService, - indicesQueryCache, codecService, - termVectorsService, indexFieldDataService, - warmer, similarityService, - factory, path, bigArrays, wrappingService); + public ShadowIndexShard(ShardId shardId, @IndexSettings Settings indexSettings, ShardPath path, Store store, IndexServicesProvider provider) throws IOException { + super(shardId, indexSettings, path, store, provider); } /** diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java index 0dbbd12834e..091985e344e 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java @@ -490,7 +490,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements public SnapshotContext(SnapshotId snapshotId, ShardId shardId, IndexShardSnapshotStatus snapshotStatus) { super(snapshotId, Version.CURRENT, shardId); IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - store = indexService.shard(shardId.id()).store(); + store = indexService.getShardOrNull(shardId.id()).store(); this.snapshotStatus = snapshotStatus; } @@ -774,7 +774,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements */ public RestoreContext(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) { super(snapshotId, version, shardId, snapshotShardId); - store = indicesService.indexServiceSafe(shardId.getIndex()).shard(shardId.id()).store(); + store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store(); this.recoveryState = recoveryState; } diff --git a/core/src/main/java/org/elasticsearch/index/store/IndexStore.java b/core/src/main/java/org/elasticsearch/index/store/IndexStore.java index 4022dd75aa1..3a23a09a652 100644 --- a/core/src/main/java/org/elasticsearch/index/store/IndexStore.java +++ b/core/src/main/java/org/elasticsearch/index/store/IndexStore.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.settings.IndexSettingsService; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.indices.store.IndicesStore; import java.io.Closeable; @@ -112,7 +113,7 @@ public class IndexStore extends AbstractIndexComponent implements Closeable { /** * The shard store class that should be used for each shard. */ - public Class shardDirectory() { - return FsDirectoryService.class; + public DirectoryService newDirectoryService(ShardPath path) { + return new FsDirectoryService(indexSettings, this, path); } } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesLifecycle.java b/core/src/main/java/org/elasticsearch/indices/IndicesLifecycle.java index 211b6d4869d..8c761dfe898 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesLifecycle.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesLifecycle.java @@ -97,17 +97,6 @@ public interface IndicesLifecycle { } - /** - * Called right after the shard is moved into POST_RECOVERY mode - */ - public void afterIndexShardPostRecovery(IndexShard indexShard) {} - - /** - * Called right before the shard is moved into POST_RECOVERY mode. - * The shard is ready to be used but not yet marked as POST_RECOVERY. - */ - public void beforeIndexShardPostRecovery(IndexShard indexShard) {} - /** * Called after the index shard has been started. */ diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index 3b24544267d..e2448670f83 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -346,7 +346,7 @@ public class IndicesService extends AbstractLifecycleComponent i modules.add(new IndexFieldDataModule(indexSettings)); modules.add(new MapperServiceModule()); modules.add(new IndexAliasesServiceModule()); - modules.add(new IndexModule(indexSettings)); + modules.add(new IndexModule()); pluginsService.processModules(modules); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java b/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java index 9ee45b21def..2a82774a612 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java @@ -87,7 +87,7 @@ public final class IndicesWarmer extends AbstractComponent { if (indexService == null) { return; } - final IndexShard indexShard = indexService.shard(context.shardId().id()); + final IndexShard indexShard = indexService.getShardOrNull(context.shardId().id()); if (indexShard == null) { return; } diff --git a/core/src/main/java/org/elasticsearch/indices/InternalIndicesLifecycle.java b/core/src/main/java/org/elasticsearch/indices/InternalIndicesLifecycle.java index 77050714db2..16c0c362c42 100644 --- a/core/src/main/java/org/elasticsearch/indices/InternalIndicesLifecycle.java +++ b/core/src/main/java/org/elasticsearch/indices/InternalIndicesLifecycle.java @@ -121,28 +121,6 @@ public class InternalIndicesLifecycle extends AbstractComponent implements Indic } } - public void beforeIndexShardPostRecovery(IndexShard indexShard) { - for (Listener listener : listeners) { - try { - listener.beforeIndexShardPostRecovery(indexShard); - } catch (Throwable t) { - logger.warn("{} failed to invoke before shard post recovery callback", t, indexShard.shardId()); - throw t; - } - } - } - - - public void afterIndexShardPostRecovery(IndexShard indexShard) { - for (Listener listener : listeners) { - try { - listener.afterIndexShardPostRecovery(indexShard); - } catch (Throwable t) { - logger.warn("{} failed to invoke after shard post recovery callback", t, indexShard.shardId()); - throw t; - } - } - } public void afterIndexShardStarted(IndexShard indexShard) { for (Listener listener : listeners) { diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java index 747d15a01f9..c8142f3d37a 100644 --- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java +++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java @@ -38,7 +38,7 @@ import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.indexing.IndexingStats; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.percolator.stats.PercolateStats; +import org.elasticsearch.index.percolator.PercolateStats; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index eb2bc242a6c..6bce5bc4be1 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -327,7 +327,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent[] asyncSendFiles(Store store, StoreFileMetaData[] files, Function outputStreamFactory) { diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java index a466147e71c..123480e81de 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java @@ -52,7 +52,7 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler { // if we relocate we need to close the engine in order to open a new // IndexWriter on the other end of the relocation engineClosed = true; - shard.engine().flushAndClose(); + shard.flushAndCloseEngine(); } catch (IOException e) { logger.warn("close engine failed", e); shard.failShard("failed to close engine (phase1)", e); diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index 45b19ae8fc5..b1cb507522e 100644 --- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -395,7 +395,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe ShardId shardId = request.shardId; IndexService indexService = indicesService.indexService(shardId.index().getName()); if (indexService != null && indexService.indexUUID().equals(request.indexUUID)) { - return indexService.shard(shardId.id()); + return indexService.getShardOrNull(shardId.id()); } return null; } diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java index 48ef0aa168c..ec5cc181aa3 100644 --- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java +++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java @@ -152,7 +152,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction percolateQueries; private final int numberOfShards; private final Query aliasFilter; private final long originNanoTime = System.nanoTime(); @@ -133,7 +134,7 @@ public class PercolateContext extends SearchContext { this.indexService = indexService; this.fieldDataService = indexService.fieldData(); this.searchShardTarget = searchShardTarget; - this.percolateQueries = indexShard.percolateRegistry().percolateQueries(); + this.percolateQueryRegistry = indexShard.percolateRegistry(); this.types = new String[]{request.documentType()}; this.pageCacheRecycler = pageCacheRecycler; this.bigArrays = bigArrays.withCircuitBreaking(); @@ -179,7 +180,7 @@ public class PercolateContext extends SearchContext { } public ConcurrentMap percolateQueries() { - return percolateQueries; + return percolateQueryRegistry.percolateQueries(); } public Query percolateQuery() { diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java index ba4ccaeb25e..b20a54f076d 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java @@ -71,7 +71,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.percolator.stats.ShardPercolateService; +import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; @@ -86,7 +86,6 @@ import org.elasticsearch.search.aggregations.AggregationPhase; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator; import org.elasticsearch.search.highlight.HighlightField; import org.elasticsearch.search.highlight.HighlightPhase; @@ -177,11 +176,10 @@ public class PercolatorService extends AbstractComponent { public PercolateShardResponse percolate(PercolateShardRequest request) { IndexService percolateIndexService = indicesService.indexServiceSafe(request.shardId().getIndex()); - IndexShard indexShard = percolateIndexService.shardSafe(request.shardId().id()); + IndexShard indexShard = percolateIndexService.getShard(request.shardId().id()); indexShard.readAllowed(); // check if we can read the shard... - - ShardPercolateService shardPercolateService = indexShard.shardPercolateService(); - shardPercolateService.prePercolate(); + PercolatorQueriesRegistry percolateQueryRegistry = indexShard.percolateRegistry(); + percolateQueryRegistry.prePercolate(); long startTime = System.nanoTime(); // TODO: The filteringAliases should be looked up at the coordinating node and serialized with all shard request, @@ -255,7 +253,7 @@ public class PercolatorService extends AbstractComponent { } finally { SearchContext.removeCurrent(); context.close(); - shardPercolateService.postPercolate(System.nanoTime() - startTime); + percolateQueryRegistry.postPercolate(System.nanoTime() - startTime); } } diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index 72077954ea8..4229c54401a 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -73,20 +73,6 @@ public abstract class Plugin { return Collections.emptyList(); } - /** - * Per index shard module. - */ - public Collection shardModules(Settings indexSettings) { - return Collections.emptyList(); - } - - /** - * Per index shard service that will be automatically closed. - */ - public Collection> shardServices() { - return Collections.emptyList(); - } - /** * Additional node settings loaded by the plugin. Note that settings that are explicit in the nodes settings can't be * overwritten with the additional settings. These settings added if they don't exist. diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index 5834efc398d..9582d3f1714 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -250,22 +250,6 @@ public class PluginsService extends AbstractComponent { return services; } - public Collection shardModules(Settings indexSettings) { - List modules = new ArrayList<>(); - for (Tuple plugin : plugins) { - modules.addAll(plugin.v2().shardModules(indexSettings)); - } - return modules; - } - - public Collection> shardServices() { - List> services = new ArrayList<>(); - for (Tuple plugin : plugins) { - services.addAll(plugin.v2().shardServices()); - } - return services; - } - /** * Get information about plugins (jvm and site plugins). */ diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 8ccf2017a81..337dd41b403 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -43,7 +43,7 @@ import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; import org.elasticsearch.index.indexing.IndexingStats; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.percolator.stats.PercolateStats; +import org.elasticsearch.index.percolator.PercolateStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.suggest.stats.SuggestStats; diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index 59ec671dd8f..403f4a5c3f6 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -559,7 +559,7 @@ public class SearchService extends AbstractLifecycleComponent { final SearchContext createContext(ShardSearchRequest request, @Nullable Engine.Searcher searcher) { IndexService indexService = indicesService.indexServiceSafe(request.index()); - IndexShard indexShard = indexService.shardSafe(request.shardId()); + IndexShard indexShard = indexService.getShard(request.shardId()); SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(), request.index(), request.shardId()); diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 3850888d848..c75189544c8 100644 --- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -289,7 +289,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent shardEntry : entry.getValue().entrySet()) { final ShardId shardId = shardEntry.getKey(); try { - final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).shard(shardId.id()); + final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()); executor.execute(new AbstractRunnable() { @Override public void doRun() { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index f040ca229a5..de9eadaf057 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -158,7 +158,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase { IndicesService indexServices = internalCluster().getInstance(IndicesService.class, node); IndexService indexShards = indexServices.indexServiceSafe(index); for (Integer shardId : indexShards.shardIds()) { - IndexShard shard = indexShards.shardSafe(shardId); + IndexShard shard = indexShards.getShard(shardId); if (randomBoolean()) { shard.failShard("test", new CorruptIndexException("test corrupted", "")); Set nodes = corruptedShardIDMap.get(shardId); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeReallyOldIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeReallyOldIndexIT.java index 4ada599f7d3..d365f5b4eeb 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeReallyOldIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/upgrade/UpgradeReallyOldIndexIT.java @@ -65,7 +65,7 @@ public class UpgradeReallyOldIndexIT extends StaticIndexBackwardCompatibilityIT for (IndicesService services : internalCluster().getInstances(IndicesService.class)) { IndexService indexService = services.indexService(index); if (indexService != null) { - assertEquals(version, indexService.shard(0).minimumCompatibleVersion()); + assertEquals(version, indexService.getShardOrNull(0).minimumCompatibleVersion()); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index 606911fae04..f672b2634bf 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -179,7 +179,7 @@ public class ClusterInfoServiceIT extends ESIntegTestCase { DiscoveryNode discoveryNode = state.getNodes().get(nodeId); IndicesService indicesService = internalTestCluster.getInstance(IndicesService.class, discoveryNode.getName()); IndexService indexService = indicesService.indexService(shard.index()); - IndexShard indexShard = indexService.shard(shard.id()); + IndexShard indexShard = indexService.getShardOrNull(shard.id()); assertEquals(indexShard.shardPath().getRootDataPath().toString(), dataPath); } diff --git a/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java b/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java index eeac5463dbb..255def77eb2 100644 --- a/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java @@ -60,6 +60,22 @@ public abstract class ModuleTestCase extends ESTestCase { fail("Did not find any binding to " + to.getName() + ". Found these bindings:\n" + s); } +// /** Configures the module and asserts "instance" is bound to "to". */ +// public void assertInstanceBinding(Module module, Class to, Object instance) { +// List elements = Elements.getElements(module); +// for (Element element : elements) { +// if (element instanceof ProviderInstanceBinding) { +// assertEquals(instance, ((ProviderInstanceBinding) element).getProviderInstance().get()); +// return; +// } +// } +// StringBuilder s = new StringBuilder(); +// for (Element element : elements) { +// s.append(element + "\n"); +// } +// fail("Did not find any binding to " + to.getName() + ". Found these bindings:\n" + s); +// } + /** * Attempts to configure the module, and asserts an {@link IllegalArgumentException} is * caught, containing the given messages @@ -164,6 +180,10 @@ public abstract class ModuleTestCase extends ESTestCase { return; } } + } else if (element instanceof ProviderInstanceBinding) { + ProviderInstanceBinding binding = (ProviderInstanceBinding) element; + assertTrue(tester.test(to.cast(binding.getProviderInstance().get()))); + return; } } StringBuilder s = new StringBuilder(); diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java new file mode 100644 index 00000000000..13957b79908 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.search.IndexSearcher; +import org.elasticsearch.common.inject.ModuleTestCase; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineException; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.shard.IndexSearcherWrapper; +import org.elasticsearch.test.engine.MockEngineFactory; + +public class IndexModuleTests extends ModuleTestCase { + + public void testWrapperIsBound() { + IndexModule module = new IndexModule(); + assertInstanceBinding(module, IndexSearcherWrapper.class,(x) -> x == null); + module.indexSearcherWrapper = Wrapper.class; + assertBinding(module, IndexSearcherWrapper.class, Wrapper.class); + } + + public void testEngineFactoryBound() { + IndexModule module = new IndexModule(); + assertBinding(module, EngineFactory.class, InternalEngineFactory.class); + module.engineFactoryImpl = MockEngineFactory.class; + assertBinding(module, EngineFactory.class, MockEngineFactory.class); + } + + public void testOtherServiceBound() { + IndexModule module = new IndexModule(); + assertBinding(module, IndexService.class, IndexService.class); + assertBinding(module, IndexServicesProvider.class, IndexServicesProvider.class); + } + + public static final class Wrapper implements IndexSearcherWrapper { + + @Override + public DirectoryReader wrap(DirectoryReader reader) { + return null; + } + + @Override + public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + return null; + } + } + +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java similarity index 67% rename from core/src/test/java/org/elasticsearch/index/shard/IndexShardModuleTests.java rename to core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index e488905710a..7d66382440a 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -17,19 +17,19 @@ * under the License. */ -package org.elasticsearch.index.shard; +package org.elasticsearch.index; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.junit.Test; -/** Unit test(s) for IndexShardModule */ -public class IndexShardModuleTests extends ESTestCase { +/** Unit test(s) for IndexService */ +public class IndexServiceTests extends ESTestCase { @Test public void testDetermineShadowEngineShouldBeUsed() { - ShardId shardId = new ShardId("myindex", 0); Settings regularSettings = Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) @@ -41,14 +41,9 @@ public class IndexShardModuleTests extends ESTestCase { .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true) .build(); - IndexShardModule ism1 = new IndexShardModule(shardId, true, regularSettings); - IndexShardModule ism2 = new IndexShardModule(shardId, false, regularSettings); - IndexShardModule ism3 = new IndexShardModule(shardId, true, shadowSettings); - IndexShardModule ism4 = new IndexShardModule(shardId, false, shadowSettings); - - assertFalse("no shadow replicas for normal settings", ism1.useShadowEngine()); - assertFalse("no shadow replicas for normal settings", ism2.useShadowEngine()); - assertFalse("no shadow replicas for primary shard with shadow settings", ism3.useShadowEngine()); - assertTrue("shadow replicas for replica shards with shadow settings", ism4.useShadowEngine()); + assertFalse("no shadow replicas for normal settings", IndexService.useShadowEngine(true, regularSettings)); + assertFalse("no shadow replicas for normal settings", IndexService.useShadowEngine(false, regularSettings)); + assertFalse("no shadow replicas for primary shard with shadow settings", IndexService.useShadowEngine(true, shadowSettings)); + assertTrue("shadow replicas for replica shards with shadow settings",IndexService.useShadowEngine(false, shadowSettings)); } } diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java index 0cbcaf9c2d3..a54be1766f8 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java @@ -150,7 +150,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { for (IndicesService service : internalCluster().getDataNodeInstances(IndicesService.class)) { if (service.hasIndex("foo-copy")) { - IndexShard shard = service.indexServiceSafe("foo-copy").shard(0); + IndexShard shard = service.indexServiceSafe("foo-copy").getShardOrNull(0); if (shard.routingEntry().primary()) { assertFalse(shard instanceof ShadowIndexShard); } else { diff --git a/core/src/test/java/org/elasticsearch/index/shard/MockEngineFactoryPlugin.java b/core/src/test/java/org/elasticsearch/index/MockEngineFactoryPlugin.java similarity index 88% rename from core/src/test/java/org/elasticsearch/index/shard/MockEngineFactoryPlugin.java rename to core/src/test/java/org/elasticsearch/index/MockEngineFactoryPlugin.java index d1b50487c63..94ddde0e3fb 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/MockEngineFactoryPlugin.java +++ b/core/src/test/java/org/elasticsearch/index/MockEngineFactoryPlugin.java @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.shard; +package org.elasticsearch.index; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.engine.MockEngineFactory; import org.elasticsearch.test.engine.MockEngineSupportModule; @@ -27,7 +28,7 @@ import org.elasticsearch.test.engine.MockEngineSupportModule; import java.util.Collection; import java.util.Collections; -// this must exist in the same package as IndexShardModule to allow access to setting the impl +// this must exist in the same package as IndexModule to allow access to setting the impl public class MockEngineFactoryPlugin extends Plugin { @Override public String name() { @@ -41,7 +42,7 @@ public class MockEngineFactoryPlugin extends Plugin { public Collection indexModules(Settings indexSettings) { return Collections.singletonList(new MockEngineSupportModule()); } - public void onModule(IndexShardModule module) { + public void onModule(IndexModule module) { module.engineFactoryImpl = MockEngineFactory.class; } } diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java index e45f1c469b0..30a8e335fda 100644 --- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -97,7 +97,7 @@ public class CodecTests extends ESSingleNodeTestCase { private static CodecService createCodecService(Settings settings) { IndexService indexService = createIndex("test", settings); - return indexService.injector().getInstance(CodecService.class); + return indexService.getIndexServices().getCodecService(); } } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java index fa5db4cdeb4..1ed022dbefa 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.engine; import org.apache.lucene.index.LiveIndexWriterConfig; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.shard.EngineAccess; import org.elasticsearch.test.ESSingleNodeTestCase; import java.util.concurrent.TimeUnit; @@ -33,7 +34,7 @@ public class InternalEngineSettingsTests extends ESSingleNodeTestCase { public void testSettingsUpdate() { final IndexService service = createIndex("foo"); // INDEX_COMPOUND_ON_FLUSH - InternalEngine engine = ((InternalEngine)engine(service)); + InternalEngine engine = ((InternalEngine) EngineAccess.engine(service.getShardOrNull(0))); assertThat(engine.getCurrentIndexWriterConfig().getUseCompoundFile(), is(true)); client().admin().indices().prepareUpdateSettings("foo").setSettings(Settings.builder().put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, false).build()).get(); assertThat(engine.getCurrentIndexWriterConfig().getUseCompoundFile(), is(false)); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 01197fbfc5b..4b7de9bc3eb 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -67,10 +67,7 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; -import org.elasticsearch.index.shard.MergeSchedulerConfig; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardUtils; -import org.elasticsearch.index.shard.TranslogRecoveryPerformer; +import org.elasticsearch.index.shard.*; import org.elasticsearch.index.similarity.SimilarityLookupService; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.DirectoryUtils; @@ -232,15 +229,15 @@ public class InternalEngineTests extends ESTestCase { return new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); } - protected InternalEngine createEngine(Store store, Path translogPath, IndexSearcherWrapper... wrappers) { - return createEngine(defaultSettings, store, translogPath, new MergeSchedulerConfig(defaultSettings), newMergePolicy(), wrappers); + protected InternalEngine createEngine(Store store, Path translogPath) { + return createEngine(defaultSettings, store, translogPath, new MergeSchedulerConfig(defaultSettings), newMergePolicy()); } - protected InternalEngine createEngine(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy, IndexSearcherWrapper... wrappers) { - return new InternalEngine(config(indexSettings, store, translogPath, mergeSchedulerConfig, mergePolicy, wrappers), false); + protected InternalEngine createEngine(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { + return new InternalEngine(config(indexSettings, store, translogPath, mergeSchedulerConfig, mergePolicy), false); } - public EngineConfig config(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy, IndexSearcherWrapper... wrappers) { + public EngineConfig config(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy) { IndexWriterConfig iwc = newIndexWriterConfig(); TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); @@ -251,7 +248,7 @@ public class InternalEngineTests extends ESTestCase { public void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable t) { // we don't need to notify anybody in this test } - }, new TranslogHandler(shardId.index().getName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), new IndexSearcherWrappingService(new HashSet<>(Arrays.asList(wrappers))), translogConfig); + }, new TranslogHandler(shardId.index().getName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig); try { config.setCreate(Lucene.indexExists(store.directory()) == false); } catch (IOException e) { @@ -491,8 +488,7 @@ public class InternalEngineTests extends ESTestCase { assertThat(stats2.getUserData(), hasKey(Translog.TRANSLOG_GENERATION_KEY)); assertThat(stats2.getUserData(), hasKey(Translog.TRANSLOG_UUID_KEY)); assertThat(stats2.getUserData().get(Translog.TRANSLOG_GENERATION_KEY), not(equalTo(stats1.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)))); - assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY))) - ; + assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY))); } @Test @@ -514,8 +510,11 @@ public class InternalEngineTests extends ESTestCase { }; Store store = createStore(); Path translog = createTempDir("translog-test"); - InternalEngine engine = createEngine(store, translog, wrapper); - Engine.Searcher searcher = engine.acquireSearcher("test"); + InternalEngine engine = createEngine(store, translog); + engine.close(); + + engine = new InternalEngine(engine.config(), false); + Engine.Searcher searcher = wrapper.wrap(engine.config(), engine.acquireSearcher("test")); assertThat(counter.get(), equalTo(2)); searcher.close(); IOUtils.close(store, engine); @@ -1951,7 +1950,7 @@ public class InternalEngineTests extends ESTestCase { EngineConfig brokenConfig = new EngineConfig(shardId, threadPool, config.getIndexingService(), config.getIndexSettings() , null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getMergeSchedulerConfig(), config.getAnalyzer(), config.getSimilarity(), new CodecService(shardId.index()), config.getFailedEngineListener() - , config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), new IndexSearcherWrappingService(), translogConfig); + , config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig); try { new InternalEngine(brokenConfig, false); diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index a6ca90a73db..b5987a92623 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -216,7 +216,7 @@ public class ShadowEngineTests extends ESTestCase { @Override public void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable t) { // we don't need to notify anybody in this test - }}, null, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), new IndexSearcherWrappingService(), translogConfig); + }}, null, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig); try { config.setCreate(Lucene.indexExists(store.directory()) == false); } catch (IOException e) { diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 5980688bfbe..94178f959a0 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -102,7 +102,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase { Settings settings = Settings.builder().put("index.fielddata.cache", "none").build(); indexService = createIndex("test", settings); mapperService = indexService.mapperService(); - indicesFieldDataCache = indexService.injector().getInstance(IndicesFieldDataCache.class); + indicesFieldDataCache = getInstanceFromNode(IndicesFieldDataCache.class); ifdService = indexService.fieldData(); // LogByteSizeMP to preserve doc ID order writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy())); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index 009bb4c7f81..a7314c2d27f 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -433,7 +433,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { client().prepareIndex(index, "type").setSource("foo", "bar").get(); client().admin().indices().prepareRefresh(index).get(); Query query = indexService.mapperService().documentMapper("type").allFieldMapper().fieldType().termQuery("bar", null); - try (Searcher searcher = indexService.shard(0).acquireSearcher("tests")) { + try (Searcher searcher = indexService.getShardOrNull(0).acquireSearcher("tests")) { query = searcher.searcher().rewrite(query); final Class expected = boost ? AllTermQuery.class : TermQuery.class; assertThat(query, Matchers.instanceOf(expected)); diff --git a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 831dc6c867a..3c3f1b44951 100644 --- a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -71,7 +71,7 @@ public class MultiMatchQueryTests extends ESSingleNodeTestCase { QueryShardContext queryShardContext = new QueryShardContext(new Index("test"), queryParser); queryShardContext.setAllowUnmappedFields(true); Query parsedQuery = multiMatchQuery("banon").field("name.first", 2).field("name.last", 3).field("foobar").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).toQuery(queryShardContext); - try (Engine.Searcher searcher = indexService.shardSafe(0).acquireSearcher("test")) { + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { Query rewrittenQuery = searcher.searcher().rewrite(parsedQuery); BooleanQuery.Builder expected = new BooleanQuery.Builder(); diff --git a/core/src/test/java/org/elasticsearch/index/shard/EngineAccess.java b/core/src/test/java/org/elasticsearch/index/shard/EngineAccess.java new file mode 100644 index 00000000000..9e5eb6c3705 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/shard/EngineAccess.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.elasticsearch.index.engine.Engine; + +/** + * Test utility to access the engine of a shard + */ +public final class EngineAccess { + + public static Engine engine(IndexShard shard) { + return shard.getEngine(); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 25a8bf2b40e..c1bdd9d2e7e 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -20,9 +20,8 @@ package org.elasticsearch.index.shard; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.Term; +import org.apache.lucene.index.*; +import org.apache.lucene.search.*; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Constants; @@ -58,13 +57,17 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexServicesProvider; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.indexing.IndexingOperationListener; import org.elasticsearch.index.indexing.ShardIndexingService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.common.ParsingException; import org.elasticsearch.index.settings.IndexSettingsService; @@ -112,7 +115,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); assertEquals(initValue, shard.isFlushOnClose()); final boolean newValue = !initValue; assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexShard.INDEX_FLUSH_ON_CLOSE, newValue).build())); @@ -183,7 +186,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals(getShardStateMetadata(shard), shardStateMetaData); ShardRouting routing = new ShardRouting(shard.shardRouting, shard.shardRouting.version() + 1); @@ -232,7 +235,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); try { shard.deleteShardState(); fail("shard is active metadata delete must fail"); @@ -259,7 +262,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); // fail shard shard.failShard("test shard fail", new CorruptIndexException("", "")); // check state file still exists @@ -304,7 +307,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexServiceSafe("test"); - IndexShard indexShard = indexService.shard(0); + IndexShard indexShard = indexService.getShardOrNull(0); client().admin().indices().prepareDelete("test").get(); assertThat(indexShard.getOperationsCount(), equalTo(0)); try { @@ -321,7 +324,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexServiceSafe("test"); - IndexShard indexShard = indexService.shard(0); + IndexShard indexShard = indexService.getShardOrNull(0); assertEquals(0, indexShard.getOperationsCount()); indexShard.incrementOperationCounter(); assertEquals(1, indexShard.getOperationsCount()); @@ -339,7 +342,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { client().prepareIndex("test", "test").setSource("{}").get(); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); - indicesService.indexService("test").shard(0).markAsInactive(); + indicesService.indexService("test").getShardOrNull(0).markAsInactive(); assertBusy(new Runnable() { // should be very very quick @Override public void run() { @@ -366,31 +369,31 @@ public class IndexShardTests extends ESSingleNodeTestCase { client().prepareIndex("test", "bar", "1").setSource("{}").get(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); setDurability(shard, Translog.Durabilty.REQUEST); - assertFalse(shard.engine().getTranslog().syncNeeded()); + assertFalse(shard.getEngine().getTranslog().syncNeeded()); setDurability(shard, Translog.Durabilty.ASYNC); client().prepareIndex("test", "bar", "2").setSource("{}").get(); - assertTrue(shard.engine().getTranslog().syncNeeded()); + assertTrue(shard.getEngine().getTranslog().syncNeeded()); setDurability(shard, Translog.Durabilty.REQUEST); client().prepareDelete("test", "bar", "1").get(); - assertFalse(shard.engine().getTranslog().syncNeeded()); + assertFalse(shard.getEngine().getTranslog().syncNeeded()); setDurability(shard, Translog.Durabilty.ASYNC); client().prepareDelete("test", "bar", "2").get(); - assertTrue(shard.engine().getTranslog().syncNeeded()); + assertTrue(shard.getEngine().getTranslog().syncNeeded()); setDurability(shard, Translog.Durabilty.REQUEST); assertNoFailures(client().prepareBulk() .add(client().prepareIndex("test", "bar", "3").setSource("{}")) .add(client().prepareDelete("test", "bar", "1")).get()); - assertFalse(shard.engine().getTranslog().syncNeeded()); + assertFalse(shard.getEngine().getTranslog().syncNeeded()); setDurability(shard, Translog.Durabilty.ASYNC); assertNoFailures(client().prepareBulk() .add(client().prepareIndex("test", "bar", "4").setSource("{}")) .add(client().prepareDelete("test", "bar", "3")).get()); setDurability(shard, Translog.Durabilty.REQUEST); - assertTrue(shard.engine().getTranslog().syncNeeded()); + assertTrue(shard.getEngine().getTranslog().syncNeeded()); } private void setDurability(IndexShard shard, Translog.Durabilty durabilty) { @@ -407,12 +410,12 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); int numDocs = 1; shard.state = IndexShardState.RECOVERING; try { shard.recoveryState().getTranslog().totalOperations(1); - shard.engine().config().getTranslogRecoveryPerformer().performRecoveryOperation(shard.engine(), new Translog.DeleteByQuery(new Engine.DeleteByQuery(null, new BytesArray("{\"term\" : { \"user\" : \"kimchy\" }}"), null, null, null, Engine.Operation.Origin.RECOVERY, 0, "person")), false); + shard.getEngine().config().getTranslogRecoveryPerformer().performRecoveryOperation(shard.getEngine(), new Translog.DeleteByQuery(new Engine.DeleteByQuery(null, new BytesArray("{\"term\" : { \"user\" : \"kimchy\" }}"), null, null, null, Engine.Operation.Origin.RECOVERY, 0, "person")), false); assertTrue(version.onOrBefore(Version.V_1_0_0_Beta2)); numDocs = 0; } catch (ParsingException ex) { @@ -420,9 +423,9 @@ public class IndexShardTests extends ESSingleNodeTestCase { } finally { shard.state = IndexShardState.STARTED; } - shard.engine().refresh("foo"); + shard.getEngine().refresh("foo"); - try (Engine.Searcher searcher = shard.engine().acquireSearcher("foo")) { + try (Engine.Searcher searcher = shard.getEngine().acquireSearcher("foo")) { assertEquals(numDocs, searcher.reader().numDocs()); } } @@ -434,11 +437,11 @@ public class IndexShardTests extends ESSingleNodeTestCase { client().prepareIndex("test", "test").setSource("{}").get(); ensureGreen("test"); IndicesService indicesService = getInstanceFromNode(IndicesService.class); - IndexShard test = indicesService.indexService("test").shard(0); + IndexShard test = indicesService.indexService("test").getShardOrNull(0); assertEquals(versionCreated.luceneVersion, test.minimumCompatibleVersion()); client().prepareIndex("test", "test").setSource("{}").get(); assertEquals(versionCreated.luceneVersion, test.minimumCompatibleVersion()); - test.engine().flush(); + test.getEngine().flush(); assertEquals(Version.CURRENT.luceneVersion, test.minimumCompatibleVersion()); } @@ -460,7 +463,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertHitCount(response, 1l); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); ShardPath shardPath = shard.shardPath(); Path dataPath = shardPath.getDataPath(); client().admin().indices().prepareClose("test").get(); @@ -580,7 +583,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(shard, new CommonStatsFlags()), shard.commitStats()); assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath()); assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath()); @@ -619,7 +622,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("testpreindex"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); ShardIndexingService shardIndexingService = shard.indexingService(); final AtomicBoolean preIndexCalled = new AtomicBoolean(false); @@ -642,7 +645,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("testpostindex"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); ShardIndexingService shardIndexingService = shard.indexingService(); final AtomicBoolean postIndexCalled = new AtomicBoolean(false); @@ -665,7 +668,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("testpostindexwithexception"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); ShardIndexingService shardIndexingService = shard.indexingService(); shard.close("Unexpected close", true); @@ -700,7 +703,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); assertFalse(shard.shouldFlush()); client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexShard.INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, 1).build()).get(); client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); @@ -709,25 +712,25 @@ public class IndexShardTests extends ESSingleNodeTestCase { Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc); shard.index(index); assertTrue(shard.shouldFlush()); - assertEquals(2, shard.engine().getTranslog().totalOperations()); + assertEquals(2, shard.getEngine().getTranslog().totalOperations()); client().prepareIndex("test", "test", "2").setSource("{}").setRefresh(randomBoolean()).get(); assertBusy(() -> { // this is async assertFalse(shard.shouldFlush()); }); - assertEquals(0, shard.engine().getTranslog().totalOperations()); - shard.engine().getTranslog().sync(); - long size = shard.engine().getTranslog().sizeInBytes(); - logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", shard.engine().getTranslog().sizeInBytes(), shard.engine().getTranslog().totalOperations(), shard.engine().getTranslog().getGeneration()); + assertEquals(0, shard.getEngine().getTranslog().totalOperations()); + shard.getEngine().getTranslog().sync(); + long size = shard.getEngine().getTranslog().sizeInBytes(); + logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration()); client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexShard.INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, 1000) .put(IndexShard.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(size, ByteSizeUnit.BYTES)) .build()).get(); client().prepareDelete("test", "test", "2").get(); - logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", shard.engine().getTranslog().sizeInBytes(), shard.engine().getTranslog().totalOperations(), shard.engine().getTranslog().getGeneration()); + logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration()); assertBusy(() -> { // this is async - logger.info("--> translog size on iter : [{}] num_ops [{}] generation [{}]", shard.engine().getTranslog().sizeInBytes(), shard.engine().getTranslog().totalOperations(), shard.engine().getTranslog().getGeneration()); + logger.info("--> translog size on iter : [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration()); assertFalse(shard.shouldFlush()); }); - assertEquals(0, shard.engine().getTranslog().totalOperations()); + assertEquals(0, shard.getEngine().getTranslog().totalOperations()); } public void testStressMaybeFlush() throws Exception { @@ -735,7 +738,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - final IndexShard shard = test.shard(0); + final IndexShard shard = test.getShardOrNull(0); assertFalse(shard.shouldFlush()); client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexShard.INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, 1).build()).get(); client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); @@ -778,7 +781,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); - final IndexShard shard = test.shard(0); + final IndexShard shard = test.getShardOrNull(0); client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); if (randomBoolean()) { @@ -804,7 +807,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); IndexService test = indicesService.indexService("test"); - final IndexShard shard = test.shard(0); + final IndexShard shard = test.getShardOrNull(0); client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); if (randomBoolean()) { @@ -852,14 +855,14 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); IndexService test_target = indicesService.indexService("test_target"); - final IndexShard test_shard = test.shard(0); + final IndexShard test_shard = test.getShardOrNull(0); client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); client().prepareIndex("test_target", "test", "1").setSource("{}").setRefresh(true).get(); assertHitCount(client().prepareSearch("test_target").get(), 1); assertSearchHits(client().prepareSearch("test_target").get(), "1"); client().admin().indices().prepareFlush("test").get(); // only flush test - final ShardRouting origRouting = test_target.shard(0).routingEntry(); + final ShardRouting origRouting = test_target.getShardOrNull(0).routingEntry(); ShardRouting routing = new ShardRouting(origRouting); ShardRoutingHelper.reinit(routing); routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(new SnapshotId("foo", "bar"), Version.CURRENT, "test")); @@ -912,10 +915,100 @@ public class IndexShardTests extends ESSingleNodeTestCase { ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService indexService = indicesService.indexService("test"); - IndexShard shard = indexService.shard(0); + IndexShard shard = indexService.getShardOrNull(0); IndexSettingsService settingsService = indexService.settingsService(); assertTrue(settingsService.isRegistered(shard)); indexService.removeShard(0, "simon says so"); assertFalse(settingsService.isRegistered(shard)); } + + public void testSearcherWrapperIsUsed() throws IOException { + createIndex("test"); + ensureGreen(); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService indexService = indicesService.indexService("test"); + IndexShard shard = indexService.getShardOrNull(0); + client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(randomBoolean()).get(); + client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefresh(true).get(); + + Engine.GetResult getResult = shard.get(new Engine.Get(false, new Term(UidFieldMapper.NAME, Uid.createUid("test", "1")))); + assertTrue(getResult.exists()); + assertNotNull(getResult.searcher()); + getResult.release(); + try (Engine.Searcher searcher = shard.acquireSearcher("test")) { + TopDocs search = searcher.searcher().search(new TermQuery(new Term("foo", "bar")), 10); + assertEquals(search.totalHits, 1); + search = searcher.searcher().search(new TermQuery(new Term("foobar", "bar")), 10); + assertEquals(search.totalHits, 1); + } + + ShardRouting routing = new ShardRouting(shard.routingEntry()); + shard.close("simon says", true); + IndexServicesProvider indexServices = indexService.getIndexServices(); + IndexSearcherWrapper wrapper = new IndexSearcherWrapper() { + @Override + public DirectoryReader wrap(DirectoryReader reader) throws IOException { + return new FieldMaskingReader("foo", reader); + } + + @Override + public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + return searcher; + } + }; + + IndexServicesProvider newProvider = new IndexServicesProvider(indexServices.getIndicesLifecycle(), indexServices.getThreadPool(), indexServices.getMapperService(), indexServices.getQueryParserService(), indexServices.getIndexCache(), indexServices.getIndexAliasesService(), indexServices.getIndicesQueryCache(), indexServices.getCodecService(), indexServices.getTermVectorsService(), indexServices.getIndexFieldDataService(), indexServices.getWarmer(), indexServices.getSimilarityService(), indexServices.getFactory(), indexServices.getBigArrays(), wrapper); + IndexShard newShard = new IndexShard(shard.shardId(), shard.indexSettings, shard.shardPath(), shard.store(), newProvider); + + ShardRoutingHelper.reinit(routing); + newShard.updateRoutingEntry(routing, false); + DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); + assertTrue(newShard.recoverFromStore(routing, localNode)); + routing = new ShardRouting(routing); + ShardRoutingHelper.moveToStarted(routing); + newShard.updateRoutingEntry(routing, true); + try (Engine.Searcher searcher = newShard.acquireSearcher("test")) { + TopDocs search = searcher.searcher().search(new TermQuery(new Term("foo", "bar")), 10); + assertEquals(search.totalHits, 0); + search = searcher.searcher().search(new TermQuery(new Term("foobar", "bar")), 10); + assertEquals(search.totalHits, 1); + } + getResult = newShard.get(new Engine.Get(false, new Term(UidFieldMapper.NAME, Uid.createUid("test", "1")))); + assertTrue(getResult.exists()); + assertNotNull(getResult.searcher()); // make sure get uses the wrapped reader + assertTrue(getResult.searcher().reader() instanceof FieldMaskingReader); + getResult.release(); + newShard.close("just do it", randomBoolean()); + } + + private static class FieldMaskingReader extends FilterDirectoryReader { + + private final String field; + public FieldMaskingReader(String field, DirectoryReader in) throws IOException { + super(in, new SubReaderWrapper() { + private final String filteredField = field; + @Override + public LeafReader wrap(LeafReader reader) { + return new FilterLeafReader(reader) { + @Override + public Fields fields() throws IOException { + return new FilterFields(super.fields()) { + @Override + public Terms terms(String field) throws IOException { + return filteredField.equals(field) ? null : super.terms(field); + } + }; + } + }; + } + }); + this.field = field; + + } + + @Override + protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { + return new FieldMaskingReader(field, in); + } + } } diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index b1ed006d695..995deaca10c 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -137,8 +137,8 @@ public class IndicesServiceTests extends ESSingleNodeTestCase { IndexService test = createIndex("test"); assertTrue(test.hasShard(0)); - ShardPath path = test.shard(0).shardPath(); - assertTrue(test.shard(0).routingEntry().started()); + ShardPath path = test.getShardOrNull(0).shardPath(); + assertTrue(test.getShardOrNull(0).routingEntry().started()); ShardPath shardPath = ShardPath.loadShardPath(logger, getNodeEnvironment(), new ShardId(test.index(), 0), test.getIndexSettings()); assertEquals(shardPath, path); try { diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java index 06c2566fc21..1a4bf8fd3f7 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java @@ -42,7 +42,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { createIndex("test"); client().prepareIndex("test", "test", "1").setSource("{}").get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class); final ShardId shardId = shard.shardId(); @@ -86,7 +86,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { createIndex("test"); client().prepareIndex("test", "test", "1").setSource("{}").get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class); final ShardId shardId = shard.shardId(); @@ -106,7 +106,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { createIndex("test"); client().prepareIndex("test", "test", "1").setSource("{}").get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class); final ShardId shardId = shard.shardId(); @@ -129,7 +129,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { public void testSyncFailsOnIndexClosedOrMissing() throws InterruptedException { createIndex("test"); IndexService test = getInstanceFromNode(IndicesService.class).indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class); SyncedFlushUtil.LatchedListener listener = new SyncedFlushUtil.LatchedListener(); @@ -162,7 +162,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { createIndex("test"); client().prepareIndex("test", "test", "1").setSource("{}").get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class); final ShardId shardId = shard.shardId(); @@ -195,7 +195,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { createIndex("test"); client().prepareIndex("test", "test", "1").setSource("{}").get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService("test"); - IndexShard shard = test.shard(0); + IndexShard shard = test.getShardOrNull(0); SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class); final ShardId shardId = shard.shardId(); diff --git a/core/src/test/java/org/elasticsearch/indices/leaks/IndicesLeaksIT.java b/core/src/test/java/org/elasticsearch/indices/leaks/IndicesLeaksIT.java deleted file mode 100644 index 422fee6879f..00000000000 --- a/core/src/test/java/org/elasticsearch/indices/leaks/IndicesLeaksIT.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.indices.leaks; - -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.junit.Test; - -import java.lang.ref.WeakReference; -import java.util.ArrayList; -import java.util.List; - -import static org.elasticsearch.test.ESIntegTestCase.Scope; -import static org.hamcrest.Matchers.nullValue; - -/** - */ -@ClusterScope(scope= Scope.TEST, numDataNodes =1) -public class IndicesLeaksIT extends ESIntegTestCase { - - - @SuppressWarnings({"ConstantConditions", "unchecked"}) - @Test - @BadApple(bugUrl = "https://github.com/elasticsearch/elasticsearch/issues/3232") - public void testIndexShardLifecycleLeak() throws Exception { - - client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) - .execute().actionGet(); - - client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); - - IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class); - IndexService indexService = indicesService.indexServiceSafe("test"); - Injector indexInjector = indexService.injector(); - IndexShard shard = indexService.shardSafe(0); - Injector shardInjector = indexService.shardInjectorSafe(0); - - performCommonOperations(); - - List indexReferences = new ArrayList<>(); - List shardReferences = new ArrayList<>(); - - // TODO if we could iterate over the already created classes on the injector, we can just add them here to the list - // for now, we simple add some classes that make sense - - // add index references - indexReferences.add(new WeakReference(indexService)); - indexReferences.add(new WeakReference(indexInjector)); - indexReferences.add(new WeakReference(indexService.mapperService())); - for (DocumentMapper documentMapper : indexService.mapperService().docMappers(true)) { - indexReferences.add(new WeakReference(documentMapper)); - } - indexReferences.add(new WeakReference(indexService.aliasesService())); - indexReferences.add(new WeakReference(indexService.analysisService())); - indexReferences.add(new WeakReference(indexService.fieldData())); - indexReferences.add(new WeakReference(indexService.queryParserService())); - - - // add shard references - shardReferences.add(new WeakReference(shard)); - shardReferences.add(new WeakReference(shardInjector)); - - indexService = null; - indexInjector = null; - shard = null; - shardInjector = null; - - cluster().wipeIndices("test"); - - for (int i = 0; i < 100; i++) { - System.gc(); - int indexNotCleared = 0; - for (WeakReference indexReference : indexReferences) { - if (indexReference.get() != null) { - indexNotCleared++; - } - } - int shardNotCleared = 0; - for (WeakReference shardReference : shardReferences) { - if (shardReference.get() != null) { - shardNotCleared++; - } - } - logger.info("round {}, indices {}/{}, shards {}/{}", i, indexNotCleared, indexReferences.size(), shardNotCleared, shardReferences.size()); - if (indexNotCleared == 0 && shardNotCleared == 0) { - break; - } - } - - //System.out.println("sleeping");Thread.sleep(1000000); - - for (WeakReference indexReference : indexReferences) { - assertThat("dangling index reference: " + indexReference.get(), indexReference.get(), nullValue()); - } - - for (WeakReference shardReference : shardReferences) { - assertThat("dangling shard reference: " + shardReference.get(), shardReference.get(), nullValue()); - } - } - - private void performCommonOperations() { - client().prepareIndex("test", "type", "1").setSource("field1", "value", "field2", 2, "field3", 3.0f).execute().actionGet(); - client().admin().indices().prepareRefresh().execute().actionGet(); - client().prepareSearch("test").setQuery(QueryBuilders.queryStringQuery("field1:value")).execute().actionGet(); - client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field1", "value")).execute().actionGet(); - } -} diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 6609ce8f66b..2eedceffcec 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -270,10 +270,10 @@ public class IndexRecoveryIT extends ESIntegTestCase { @Override public void run() { IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA); - assertThat(indicesService.indexServiceSafe(INDEX_NAME).shardSafe(0).recoveryStats().currentAsSource(), + assertThat(indicesService.indexServiceSafe(INDEX_NAME).getShard(0).recoveryStats().currentAsSource(), equalTo(1)); indicesService = internalCluster().getInstance(IndicesService.class, nodeB); - assertThat(indicesService.indexServiceSafe(INDEX_NAME).shardSafe(0).recoveryStats().currentAsTarget(), + assertThat(indicesService.indexServiceSafe(INDEX_NAME).getShard(0).recoveryStats().currentAsTarget(), equalTo(1)); } }); diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index ed73a44c517..edb0f7b6a78 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -39,7 +39,7 @@ public class RecoveryStatusTests extends ESSingleNodeTestCase { public void testRenameTempFiles() throws IOException { IndexService service = createIndex("foo"); - IndexShard indexShard = service.shard(0); + IndexShard indexShard = service.getShardOrNull(0); DiscoveryNode node = new DiscoveryNode("foo", new LocalTransportAddress("bar"), Version.CURRENT); RecoveryStatus status = new RecoveryStatus(indexShard, node, new RecoveryTarget.RecoveryListener() { @Override diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java index b1d97b8b4ab..5a5f3163a4c 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java @@ -171,7 +171,7 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase { long startRecovery(RecoveriesCollection collection, RecoveryTarget.RecoveryListener listener, TimeValue timeValue) { IndicesService indexServices = getInstanceFromNode(IndicesService.class); - IndexShard indexShard = indexServices.indexServiceSafe("test").shard(0); + IndexShard indexShard = indexServices.indexServiceSafe("test").getShardOrNull(0); final DiscoveryNode sourceNode = new DiscoveryNode("id", DummyTransportAddress.INSTANCE, Version.CURRENT); return collection.startRecovery(indexShard, sourceNode, listener, timeValue); } diff --git a/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 76602a1ab55..c877da877d4 100644 --- a/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -41,6 +41,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -215,18 +216,15 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { return instanceFromNode.indexServiceSafe(index); } - protected static org.elasticsearch.index.engine.Engine engine(IndexService service) { - return service.shard(0).engine(); - } - /** * Create a new search context. */ protected static SearchContext createSearchContext(IndexService indexService) { - BigArrays bigArrays = indexService.injector().getInstance(BigArrays.class); - ThreadPool threadPool = indexService.injector().getInstance(ThreadPool.class); - PageCacheRecycler pageCacheRecycler = indexService.injector().getInstance(PageCacheRecycler.class); - return new TestSearchContext(threadPool, pageCacheRecycler, bigArrays, indexService); + BigArrays bigArrays = indexService.getIndexServices().getBigArrays(); + ThreadPool threadPool = indexService.getIndexServices().getThreadPool(); + PageCacheRecycler pageCacheRecycler = node().injector().getInstance(PageCacheRecycler.class); + ScriptService scriptService = node().injector().getInstance(ScriptService.class); + return new TestSearchContext(threadPool, pageCacheRecycler, bigArrays, scriptService, indexService); } /** diff --git a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java index 53733bca3e3..f7c44a55ff7 100644 --- a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -68,7 +68,7 @@ import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineClosedException; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.MockEngineFactoryPlugin; +import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -1047,8 +1047,8 @@ public final class InternalTestCluster extends TestCluster { IndicesService indexServices = getInstance(IndicesService.class, nodeAndClient.name); for (IndexService indexService : indexServices) { for (IndexShard indexShard : indexService) { - try { - CommitStats commitStats = indexShard.engine().commitStats(); + CommitStats commitStats = indexShard.commitStats(); + if (commitStats != null) { // null if the engine is closed or if the shard is recovering String syncId = commitStats.getUserData().get(Engine.SYNC_COMMIT_ID); if (syncId != null) { long liveDocsOnShard = commitStats.getNumDocs(); @@ -1058,8 +1058,6 @@ public final class InternalTestCluster extends TestCluster { docsOnShards.put(syncId, liveDocsOnShard); } } - } catch (EngineClosedException e) { - // nothing to do, shard is closed } } } @@ -1741,7 +1739,7 @@ public final class InternalTestCluster extends TestCluster { IndexService indexService = indicesService.indexService(index); if (indexService != null) { assertThat(indexService.settingsService().getSettings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), greaterThan(shard)); - OperationRouting operationRouting = indexService.injector().getInstance(OperationRouting.class); + OperationRouting operationRouting = getInstanceFromNode(OperationRouting.class, node); while (true) { String routing = RandomStrings.randomAsciiOfLength(random, 10); final int targetShard = operationRouting.indexShards(clusterService.state(), index, type, null, routing).shardId().getId(); diff --git a/core/src/test/java/org/elasticsearch/test/TestSearchContext.java b/core/src/test/java/org/elasticsearch/test/TestSearchContext.java index 4ee8ac6c6ec..35b6ab2f835 100644 --- a/core/src/test/java/org/elasticsearch/test/TestSearchContext.java +++ b/core/src/test/java/org/elasticsearch/test/TestSearchContext.java @@ -85,6 +85,7 @@ public class TestSearchContext extends SearchContext { final IndexShard indexShard; final Counter timeEstimateCounter = Counter.newCounter(); final QuerySearchResult queryResult = new QuerySearchResult(); + ScriptService scriptService; ParsedQuery originalQuery; ParsedQuery postFilter; Query query; @@ -99,7 +100,7 @@ public class TestSearchContext extends SearchContext { private final long originNanoTime = System.nanoTime(); private final Map subPhaseContexts = new HashMap<>(); - public TestSearchContext(ThreadPool threadPool,PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, IndexService indexService) { + public TestSearchContext(ThreadPool threadPool,PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, ScriptService scriptService, IndexService indexService) { super(ParseFieldMatcher.STRICT, null); this.pageCacheRecycler = pageCacheRecycler; this.bigArrays = bigArrays.withCircuitBreaking(); @@ -107,7 +108,8 @@ public class TestSearchContext extends SearchContext { this.indexFieldDataService = indexService.fieldData(); this.fixedBitSetFilterCache = indexService.bitsetFilterCache(); this.threadPool = threadPool; - this.indexShard = indexService.shard(0); + this.indexShard = indexService.getShardOrNull(0); + this.scriptService = scriptService; } public TestSearchContext() { @@ -119,6 +121,7 @@ public class TestSearchContext extends SearchContext { this.threadPool = null; this.fixedBitSetFilterCache = null; this.indexShard = null; + scriptService = null; } public void setTypes(String... types) { @@ -325,7 +328,7 @@ public class TestSearchContext extends SearchContext { @Override public ScriptService scriptService() { - return indexService.injector().getInstance(ScriptService.class); + return scriptService; } @Override diff --git a/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java b/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java index c5b5ac36f00..11a791c04f3 100644 --- a/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java +++ b/core/src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java @@ -24,14 +24,19 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.settings.IndexSettingsService; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.IndexStoreModule; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.plugins.Plugin; public class MockFSIndexStore extends IndexStore { + private final IndicesService indicesService; + public static class TestPlugin extends Plugin { @Override public String name() { @@ -52,13 +57,13 @@ public class MockFSIndexStore extends IndexStore { @Inject public MockFSIndexStore(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService, - IndicesStore indicesStore) { + IndicesStore indicesStore, IndicesService indicesService) { super(index, indexSettings, indexSettingsService, indicesStore); + this.indicesService = indicesService; } - @Override - public Class shardDirectory() { - return MockFSDirectoryService.class; + public DirectoryService newDirectoryService(ShardPath path) { + return new MockFSDirectoryService(indexSettings, this, indicesService, path); } } diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java index fc9de8912d6..9c4ec733a9f 100644 --- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java +++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java @@ -66,24 +66,10 @@ public class JvmExamplePlugin extends Plugin { } @Override - public Collection indexModules(Settings indexSettings) { - return Collections.emptyList(); - } + public Collection indexModules(Settings indexSettings) { return Collections.emptyList();} @Override - public Collection> indexServices() { - return Collections.emptyList(); - } - - @Override - public Collection shardModules(Settings indexSettings) { - return Collections.emptyList(); - } - - @Override - public Collection> shardServices() { - return Collections.emptyList(); - } + public Collection> indexServices() { return Collections.emptyList();} @Override public Settings additionalSettings() { diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java index 04229756e1b..1d1592dcf32 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsIndexStore.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.settings.IndexSettingsService; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.indices.store.IndicesStore; @@ -37,7 +38,7 @@ public class SmbMmapFsIndexStore extends IndexStore { } @Override - public Class shardDirectory() { - return SmbMmapFsDirectoryService.class; + public DirectoryService newDirectoryService(ShardPath path) { + return new SmbMmapFsDirectoryService(indexSettings(), this, path); } } diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java index 48133440f6d..67d396a80a5 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbsimplefs/SmbSimpleFsIndexStore.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.settings.IndexSettingsService; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.indices.store.IndicesStore; @@ -36,9 +37,13 @@ public class SmbSimpleFsIndexStore extends IndexStore { super(index, indexSettings, indexSettingsService, indicesStore); } - @Override public Class shardDirectory() { return SmbSimpleFsDirectoryService.class; } + + @Override + public DirectoryService newDirectoryService(ShardPath path) { + return new SmbSimpleFsDirectoryService(indexSettings(), this, path); + } }