From a892a35f403de0eecdcf694d03c0c2e1a817d6ad Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 1 Oct 2015 20:00:10 +0200 Subject: [PATCH] Hide engine entirely in IndexShard and do searcher wrapping only on top of the engine --- .../TransportIndicesSegmentsAction.java | 2 +- .../get/TransportUpgradeStatusAction.java | 2 +- .../org/elasticsearch/index/IndexModule.java | 12 ++- .../org/elasticsearch/index/IndexService.java | 17 ---- .../index/IndexServicesProvider.java | 11 ++- .../elasticsearch/index/engine/Engine.java | 37 +-------- .../index/engine/EngineConfig.java | 9 +-- .../index/engine/IndexSearcherWrapper.java | 47 ----------- .../percolator/PercolatorQueriesRegistry.java | 2 +- .../index/shard/IndexSearcherWrapper.java | 79 +++++++++++++++++++ .../elasticsearch/index/shard/IndexShard.java | 35 ++++++-- .../memory/IndexingMemoryController.java | 2 +- .../recovery/RecoverySourceHandler.java | 10 +-- .../SharedFSRecoverySourceHandler.java | 2 +- .../engine/InternalEngineSettingsTests.java | 3 +- .../index/engine/InternalEngineTests.java | 12 +-- .../index/shard/EngineAccess.java | 31 ++++++++ .../test/ESSingleNodeTestCase.java | 4 - .../test/InternalTestCluster.java | 2 +- 19 files changed, 177 insertions(+), 142 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java create mode 100644 core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java create mode 100644 core/src/test/java/org/elasticsearch/index/shard/EngineAccess.java diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java index e7770a52aee..bac0ff91b3e 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java @@ -95,6 +95,6 @@ public class TransportIndicesSegmentsAction extends TransportBroadcastByNodeActi protected ShardSegments shardOperation(IndicesSegmentsRequest request, ShardRouting shardRouting) { IndexService indexService = indicesService.indexServiceSafe(shardRouting.getIndex()); IndexShard indexShard = indexService.shardSafe(shardRouting.id()); - return new ShardSegments(indexShard.routingEntry(), indexShard.engine().segments(request.verbose())); + return new ShardSegments(indexShard.routingEntry(), indexShard.segments(request.verbose())); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java index ea2a2ede6c6..c1408e1bd80 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/TransportUpgradeStatusAction.java @@ -97,7 +97,7 @@ public class TransportUpgradeStatusAction extends TransportBroadcastByNodeAction protected ShardUpgradeStatus shardOperation(UpgradeStatusRequest request, ShardRouting shardRouting) { IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.shardSafe(shardRouting.shardId().id()); - List segments = indexShard.engine().segments(false); + List segments = indexShard.segments(false); long total_bytes = 0; long to_upgrade_bytes = 0; long to_upgrade_bytes_ancient = 0; diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java index 59bec88d81a..0c70dd456ca 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java @@ -20,8 +20,10 @@ package org.elasticsearch.index; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.util.Providers; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.shard.IndexSearcherWrapper; /** * @@ -30,11 +32,19 @@ public class IndexModule extends AbstractModule { // pkg private so tests can mock Class engineFactoryImpl = InternalEngineFactory.class; + Class indexSearcherWrapper = null; @Override protected void configure() { - bind(EngineFactory.class).to(engineFactoryImpl); + bind(EngineFactory.class).to(engineFactoryImpl).asEagerSingleton(); + if (indexSearcherWrapper == null) { + bind(IndexSearcherWrapper.class).toProvider(Providers.of(null)); + } else { + bind(IndexSearcherWrapper.class).to(indexSearcherWrapper).asEagerSingleton(); + } bind(IndexService.class).asEagerSingleton(); bind(IndexServicesProvider.class).asEagerSingleton(); } + + } diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index b48b2aedeae..22419ba23cf 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -390,23 +390,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone } } - /** - * Closes an optional resource. Returns true if the resource was found; - * NOTE: this method swallows all exceptions thrown from the close method of the injector and logs them as debug log - */ - private boolean closeInjectorOptionalResource(ShardId shardId, Injector shardInjector, Class toClose) { - try { - final Closeable instance = shardInjector.getInstance(toClose); - if (instance == null) { - return false; - } - IOUtils.close(instance); - } catch (Throwable t) { - logger.debug("{} failed to close {}", t, shardId, Strings.toUnderscoreCase(toClose.getSimpleName())); - } - return true; - } - private void onShardClose(ShardLock lock, boolean ownsShard) { if (deleted.get()) { // we remove that shards content if this index has been deleted diff --git a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java index ad136c58dd6..0a34fabd7b1 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java +++ b/core/src/main/java/org/elasticsearch/index/IndexServicesProvider.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.aliases.IndexAliasesService; @@ -27,6 +28,7 @@ import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.IndexQueryParserService; +import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.indices.IndicesLifecycle; @@ -56,8 +58,12 @@ public final class IndexServicesProvider { private final EngineFactory factory; private final BigArrays bigArrays; + + + private final IndexSearcherWrapper indexSearcherWrapper; + @Inject - public IndexServicesProvider(IndicesLifecycle indicesLifecycle, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays) { + public IndexServicesProvider(IndicesLifecycle indicesLifecycle, ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, IndicesQueryCache indicesQueryCache, CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, @Nullable IndicesWarmer warmer, SimilarityService similarityService, EngineFactory factory, BigArrays bigArrays, @Nullable IndexSearcherWrapper indexSearcherWrapper) { this.indicesLifecycle = indicesLifecycle; this.threadPool = threadPool; this.mapperService = mapperService; @@ -72,6 +78,7 @@ public final class IndexServicesProvider { this.similarityService = similarityService; this.factory = factory; this.bigArrays = bigArrays; + this.indexSearcherWrapper = indexSearcherWrapper; } public IndicesLifecycle getIndicesLifecycle() { @@ -129,4 +136,6 @@ public final class IndexServicesProvider { public BigArrays getBigArrays() { return bigArrays; } + + public IndexSearcherWrapper getIndexSearcherWrapper() { return indexSearcherWrapper; } } diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index dc35b95d2c1..ce5f5178cfe 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -79,8 +79,6 @@ public abstract class Engine implements Closeable { protected final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); protected final ReleasableLock readLock = new ReleasableLock(rwl.readLock()); protected final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock()); - private final IndexSearcherWrapper searcherWrapper; - protected volatile Throwable failedEngine = null; protected Engine(EngineConfig engineConfig) { @@ -94,7 +92,6 @@ public abstract class Engine implements Closeable { engineConfig.getIndexSettings(), engineConfig.getShardId()); this.failedEngineListener = engineConfig.getFailedEngineListener(); this.deletionPolicy = engineConfig.getDeletionPolicy(); - this.searcherWrapper = engineConfig.getSearcherWrapper(); } /** Returns 0 in the case where accountable is null, otherwise returns {@code ramBytesUsed()} */ @@ -282,7 +279,7 @@ public abstract class Engine implements Closeable { try { final Searcher retVal = newSearcher(source, searcher, manager); success = true; - return wrap(engineConfig, retVal); + return retVal; } finally { if (!success) { manager.release(searcher); @@ -301,38 +298,6 @@ public abstract class Engine implements Closeable { } } - /** - * If there are configured {@link IndexSearcherWrapper} instances, the {@link IndexSearcher} of the provided engine searcher - * gets wrapped and a new {@link Searcher} instances is returned, otherwise the provided {@link Searcher} is returned. - * - * This is invoked each time a {@link Searcher} is requested to do an operation. (for example search) - */ - private Searcher wrap(EngineConfig engineConfig, final Searcher engineSearcher) throws EngineException { - if (searcherWrapper == null) { - return engineSearcher; - } - - DirectoryReader reader = searcherWrapper.wrap((DirectoryReader) engineSearcher.reader()); - IndexSearcher innerIndexSearcher = new IndexSearcher(reader); - innerIndexSearcher.setQueryCache(engineConfig.getQueryCache()); - innerIndexSearcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy()); - innerIndexSearcher.setSimilarity(engineConfig.getSimilarity()); - // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point - // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten - // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times - IndexSearcher indexSearcher = searcherWrapper.wrap(engineConfig, innerIndexSearcher); - if (reader == engineSearcher.reader() && indexSearcher == innerIndexSearcher) { - return engineSearcher; - } else { - return new Engine.Searcher(engineSearcher.source(), indexSearcher) { - @Override - public void close() throws ElasticsearchException { - engineSearcher.close(); - } - }; - } - } - /** returns the translog for this engine */ public abstract Translog getTranslog(); diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 7d57bb5b0b3..a79587e4347 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.indexing.ShardIndexingService; +import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.TranslogRecoveryPerformer; @@ -380,14 +381,6 @@ public final class EngineConfig { return queryCachingPolicy; } - IndexSearcherWrapper getSearcherWrapper() { - return searcherWrapper.get(); - } - - public void setSearcherWrapper(IndexSearcherWrapper searcherWrapper) { - this.searcherWrapper.set(searcherWrapper); - } - /** * Returns the translog config for this engine */ diff --git a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java deleted file mode 100644 index 8a407f00eae..00000000000 --- a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.engine; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.search.IndexSearcher; - -/** - * Extension point to add custom functionality at request time to the {@link DirectoryReader} - * and {@link IndexSearcher} managed by the {@link Engine}. - */ -interface IndexSearcherWrapper { - - /** - * @param reader The provided directory reader to be wrapped to add custom functionality - * @return a new directory reader wrapping the provided directory reader or if no wrapping was performed - * the provided directory reader - */ - DirectoryReader wrap(DirectoryReader reader); - - /** - * @param engineConfig The engine config which can be used to get the query cache and query cache policy from - * when creating a new index searcher - * @param searcher The provided index searcher to be wrapped to add custom functionality - * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed - * the provided index searcher - */ - IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException; - -} diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java index 7dd26ec55db..22f2b3cbe44 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java @@ -257,7 +257,7 @@ public class PercolatorQueriesRegistry extends AbstractIndexShardComponent imple shard.refresh("percolator_load_queries"); // NOTE: we acquire the searcher via the engine directly here since this is executed right // before the shard is marked as POST_RECOVERY - try (Engine.Searcher searcher = shard.engine().acquireSearcher("percolator_load_queries")) { + try (Engine.Searcher searcher = shard.acquireSearcher("percolator_load_queries")) { Query query = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME)); QueriesLoaderCollector queryCollector = new QueriesLoaderCollector(PercolatorQueriesRegistry.this, logger, mapperService, indexFieldDataService); IndexSearcher indexSearcher = new IndexSearcher(searcher.reader()); diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java new file mode 100644 index 00000000000..9bc51f6f57b --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.search.IndexSearcher; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineException; + +/** + * Extension point to add custom functionality at request time to the {@link DirectoryReader} + * and {@link IndexSearcher} managed by the {@link Engine}. + */ +public interface IndexSearcherWrapper { + + /** + * @param reader The provided directory reader to be wrapped to add custom functionality + * @return a new directory reader wrapping the provided directory reader or if no wrapping was performed + * the provided directory reader + */ + DirectoryReader wrap(DirectoryReader reader); + + /** + * @param engineConfig The engine config which can be used to get the query cache and query cache policy from + * when creating a new index searcher + * @param searcher The provided index searcher to be wrapped to add custom functionality + * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed + * the provided index searcher + */ + IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException; + + /** + * If there are configured {@link IndexSearcherWrapper} instances, the {@link IndexSearcher} of the provided engine searcher + * gets wrapped and a new {@link Engine.Searcher} instances is returned, otherwise the provided {@link Engine.Searcher} is returned. + * + * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search) + */ + default Engine.Searcher wrap(EngineConfig engineConfig, Engine.Searcher engineSearcher) { + DirectoryReader reader = wrap((DirectoryReader) engineSearcher.reader()); + IndexSearcher innerIndexSearcher = new IndexSearcher(reader); + innerIndexSearcher.setQueryCache(engineConfig.getQueryCache()); + innerIndexSearcher.setQueryCachingPolicy(engineConfig.getQueryCachingPolicy()); + innerIndexSearcher.setSimilarity(engineConfig.getSimilarity()); + // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point + // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten + // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times + IndexSearcher indexSearcher = wrap(engineConfig, innerIndexSearcher); + if (reader == engineSearcher.reader() && indexSearcher == innerIndexSearcher) { + return engineSearcher; + } else { + return new Engine.Searcher(engineSearcher.source(), indexSearcher) { + @Override + public void close() throws ElasticsearchException { + engineSearcher.close(); + } + }; + } + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 2137a42c658..ec5db1db46a 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -20,10 +20,8 @@ package org.elasticsearch.index.shard; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.index.CheckIndex; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; -import org.apache.lucene.index.SnapshotDeletionPolicy; +import org.apache.lucene.index.*; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.store.AlreadyClosedException; @@ -188,7 +186,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett private final IndexShardOperationCounter indexShardOperationCounter; - private EnumSet readAllowedStates = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY); + private final EnumSet readAllowedStates = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY); + + private final IndexSearcherWrapper searcherWrapper; @Inject public IndexShard(ShardId shardId, @IndexSettings Settings indexSettings, ShardPath path, Store store, IndexServicesProvider provider) { @@ -244,6 +244,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett this.flushThresholdSize = indexSettings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB)); this.disableFlush = indexSettings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false); this.indexShardOperationCounter = new IndexShardOperationCounter(logger, shardId); + this.searcherWrapper = provider.getIndexSearcherWrapper(); } public Store store() { @@ -739,7 +740,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett public Engine.Searcher acquireSearcher(String source) { readAllowed(); - return engine().acquireSearcher(source); + Engine engine = engine(); + return searcherWrapper == null ? engine.acquireSearcher(source) : searcherWrapper.wrap(engineConfig, engine.acquireSearcher(source)); } public void close(String reason, boolean flushEngine) throws IOException { @@ -1167,6 +1169,24 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } } + public Translog.View acquireTranslogView() { + Engine engine = engine(); + assert engine.getTranslog() != null : "translog must not be null"; + return engine.getTranslog().newView(); + } + + public List segments(boolean verbose) { + return engine().segments(verbose); + } + + public void flushAndCloseEngine() throws IOException { + engine().flushAndClose(); + } + + public Translog getTranslog() { + return engine().getTranslog(); + } + class EngineRefresher implements Runnable { @Override public void run() { @@ -1292,7 +1312,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett recoveryState.getVerifyIndex().checkIndexTime(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - timeNS))); } - public Engine engine() { + Engine engine() { Engine engine = engineUnsafe(); if (engine == null) { throw new EngineClosedException(shardId); @@ -1507,4 +1527,5 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett } return false; } + } diff --git a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java index 8703ab2b10a..5acef57cd6f 100644 --- a/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryController.java @@ -264,7 +264,7 @@ public class IndexingMemoryController extends AbstractLifecycleComponent[] asyncSendFiles(Store store, StoreFileMetaData[] files, Function outputStreamFactory) { diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java index a466147e71c..123480e81de 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java @@ -52,7 +52,7 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler { // if we relocate we need to close the engine in order to open a new // IndexWriter on the other end of the relocation engineClosed = true; - shard.engine().flushAndClose(); + shard.flushAndCloseEngine(); } catch (IOException e) { logger.warn("close engine failed", e); shard.failShard("failed to close engine (phase1)", e); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java index fa5db4cdeb4..78705f54a91 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.engine; import org.apache.lucene.index.LiveIndexWriterConfig; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.shard.EngineAccess; import org.elasticsearch.test.ESSingleNodeTestCase; import java.util.concurrent.TimeUnit; @@ -33,7 +34,7 @@ public class InternalEngineSettingsTests extends ESSingleNodeTestCase { public void testSettingsUpdate() { final IndexService service = createIndex("foo"); // INDEX_COMPOUND_ON_FLUSH - InternalEngine engine = ((InternalEngine)engine(service)); + InternalEngine engine = ((InternalEngine) EngineAccess.engine(service.shard(0))); assertThat(engine.getCurrentIndexWriterConfig().getUseCompoundFile(), is(true)); client().admin().indices().prepareUpdateSettings("foo").setSettings(Settings.builder().put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, false).build()).get(); assertThat(engine.getCurrentIndexWriterConfig().getUseCompoundFile(), is(false)); diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 62964244344..4b7de9bc3eb 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -67,10 +67,7 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; -import org.elasticsearch.index.shard.MergeSchedulerConfig; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.shard.ShardUtils; -import org.elasticsearch.index.shard.TranslogRecoveryPerformer; +import org.elasticsearch.index.shard.*; import org.elasticsearch.index.similarity.SimilarityLookupService; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.DirectoryUtils; @@ -491,8 +488,7 @@ public class InternalEngineTests extends ESTestCase { assertThat(stats2.getUserData(), hasKey(Translog.TRANSLOG_GENERATION_KEY)); assertThat(stats2.getUserData(), hasKey(Translog.TRANSLOG_UUID_KEY)); assertThat(stats2.getUserData().get(Translog.TRANSLOG_GENERATION_KEY), not(equalTo(stats1.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)))); - assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY))) - ; + assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY))); } @Test @@ -516,9 +512,9 @@ public class InternalEngineTests extends ESTestCase { Path translog = createTempDir("translog-test"); InternalEngine engine = createEngine(store, translog); engine.close(); - engine.config().setSearcherWrapper(wrapper); + engine = new InternalEngine(engine.config(), false); - Engine.Searcher searcher = engine.acquireSearcher("test"); + Engine.Searcher searcher = wrapper.wrap(engine.config(), engine.acquireSearcher("test")); assertThat(counter.get(), equalTo(2)); searcher.close(); IOUtils.close(store, engine); diff --git a/core/src/test/java/org/elasticsearch/index/shard/EngineAccess.java b/core/src/test/java/org/elasticsearch/index/shard/EngineAccess.java new file mode 100644 index 00000000000..58e4ddb67a5 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/shard/EngineAccess.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.elasticsearch.index.engine.Engine; + +/** + * Test utility to access the engine of a shard + */ +public final class EngineAccess { + + public static Engine engine(IndexShard shard) { + return shard.engine(); + } +} diff --git a/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 76602a1ab55..50714e1ed31 100644 --- a/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -215,10 +215,6 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { return instanceFromNode.indexServiceSafe(index); } - protected static org.elasticsearch.index.engine.Engine engine(IndexService service) { - return service.shard(0).engine(); - } - /** * Create a new search context. */ diff --git a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java index 4d107d1151d..ab273000ed6 100644 --- a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -1048,7 +1048,7 @@ public final class InternalTestCluster extends TestCluster { for (IndexService indexService : indexServices) { for (IndexShard indexShard : indexService) { try { - CommitStats commitStats = indexShard.engine().commitStats(); + CommitStats commitStats = indexShard.commitStats(); String syncId = commitStats.getUserData().get(Engine.SYNC_COMMIT_ID); if (syncId != null) { long liveDocsOnShard = commitStats.getNumDocs();