From 4676eb19a4c622bd7b9a74c27cc9888744377150 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 5 Oct 2015 10:27:34 +0200 Subject: [PATCH] add tests for IndexSearcherWrapper --- .../index/shard/IndexSearcherWrapper.java | 8 +- .../elasticsearch/index/shard/IndexShard.java | 7 +- .../common/inject/ModuleTestCase.java | 20 ++++ .../elasticsearch/index/IndexModuleTests.java | 66 +++++++++++++ .../index/shard/IndexShardTests.java | 93 ++++++++++++++++++- 5 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/index/IndexModuleTests.java diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java index 9bc51f6f57b..c75f3c7995f 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -26,6 +26,8 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.engine.EngineException; +import java.io.IOException; + /** * Extension point to add custom functionality at request time to the {@link DirectoryReader} * and {@link IndexSearcher} managed by the {@link Engine}. @@ -37,7 +39,7 @@ public interface IndexSearcherWrapper { * @return a new directory reader wrapping the provided directory reader or if no wrapping was performed * the provided directory reader */ - DirectoryReader wrap(DirectoryReader reader); + DirectoryReader wrap(DirectoryReader reader) throws IOException; /** * @param engineConfig The engine config which can be used to get the query cache and query cache policy from @@ -46,7 +48,7 @@ public interface IndexSearcherWrapper { * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed * the provided index searcher */ - IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException; + IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws IOException; /** * If there are configured {@link IndexSearcherWrapper} instances, the {@link IndexSearcher} of the provided engine searcher @@ -54,7 +56,7 @@ public interface IndexSearcherWrapper { * * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search) */ - default Engine.Searcher wrap(EngineConfig engineConfig, Engine.Searcher engineSearcher) { + default Engine.Searcher wrap(EngineConfig engineConfig, Engine.Searcher engineSearcher) throws IOException { DirectoryReader reader = wrap((DirectoryReader) engineSearcher.reader()); IndexSearcher innerIndexSearcher = new IndexSearcher(reader); innerIndexSearcher.setQueryCache(engineConfig.getQueryCache()); diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index d0fbd1873f1..ea2d555ae0d 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -33,6 +33,7 @@ import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest; import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; +import org.elasticsearch.bootstrap.Elasticsearch; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -738,7 +739,11 @@ public class IndexShard extends AbstractIndexShardComponent implements IndexSett public Engine.Searcher acquireSearcher(String source) { readAllowed(); Engine engine = getEngine(); - return searcherWrapper == null ? engine.acquireSearcher(source) : searcherWrapper.wrap(engineConfig, engine.acquireSearcher(source)); + try { + return searcherWrapper == null ? engine.acquireSearcher(source) : searcherWrapper.wrap(engineConfig, engine.acquireSearcher(source)); + } catch (IOException ex) { + throw new ElasticsearchException("failed to wrap searcher", ex); + } } public void close(String reason, boolean flushEngine) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java b/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java index eeac5463dbb..255def77eb2 100644 --- a/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/inject/ModuleTestCase.java @@ -60,6 +60,22 @@ public abstract class ModuleTestCase extends ESTestCase { fail("Did not find any binding to " + to.getName() + ". Found these bindings:\n" + s); } +// /** Configures the module and asserts "instance" is bound to "to". */ +// public void assertInstanceBinding(Module module, Class to, Object instance) { +// List elements = Elements.getElements(module); +// for (Element element : elements) { +// if (element instanceof ProviderInstanceBinding) { +// assertEquals(instance, ((ProviderInstanceBinding) element).getProviderInstance().get()); +// return; +// } +// } +// StringBuilder s = new StringBuilder(); +// for (Element element : elements) { +// s.append(element + "\n"); +// } +// fail("Did not find any binding to " + to.getName() + ". Found these bindings:\n" + s); +// } + /** * Attempts to configure the module, and asserts an {@link IllegalArgumentException} is * caught, containing the given messages @@ -164,6 +180,10 @@ public abstract class ModuleTestCase extends ESTestCase { return; } } + } else if (element instanceof ProviderInstanceBinding) { + ProviderInstanceBinding binding = (ProviderInstanceBinding) element; + assertTrue(tester.test(to.cast(binding.getProviderInstance().get()))); + return; } } StringBuilder s = new StringBuilder(); diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java new file mode 100644 index 00000000000..13957b79908 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.search.IndexSearcher; +import org.elasticsearch.common.inject.ModuleTestCase; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineException; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.shard.IndexSearcherWrapper; +import org.elasticsearch.test.engine.MockEngineFactory; + +public class IndexModuleTests extends ModuleTestCase { + + public void testWrapperIsBound() { + IndexModule module = new IndexModule(); + assertInstanceBinding(module, IndexSearcherWrapper.class,(x) -> x == null); + module.indexSearcherWrapper = Wrapper.class; + assertBinding(module, IndexSearcherWrapper.class, Wrapper.class); + } + + public void testEngineFactoryBound() { + IndexModule module = new IndexModule(); + assertBinding(module, EngineFactory.class, InternalEngineFactory.class); + module.engineFactoryImpl = MockEngineFactory.class; + assertBinding(module, EngineFactory.class, MockEngineFactory.class); + } + + public void testOtherServiceBound() { + IndexModule module = new IndexModule(); + assertBinding(module, IndexService.class, IndexService.class); + assertBinding(module, IndexServicesProvider.class, IndexServicesProvider.class); + } + + public static final class Wrapper implements IndexSearcherWrapper { + + @Override + public DirectoryReader wrap(DirectoryReader reader) { + return null; + } + + @Override + public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + return null; + } + } + +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 41aff8f4103..eb46f2d4e2a 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -20,9 +20,10 @@ package org.elasticsearch.index.shard; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.index.Term; +import org.apache.lucene.index.*; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Constants; @@ -58,7 +59,10 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexServicesProvider; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.indexing.IndexingOperationListener; import org.elasticsearch.index.indexing.ShardIndexingService; @@ -918,4 +922,87 @@ public class IndexShardTests extends ESSingleNodeTestCase { indexService.removeShard(0, "simon says so"); assertFalse(settingsService.isRegistered(shard)); } + + public void testSearcherWrapperIsUsed() throws IOException { + createIndex("test"); + ensureGreen(); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService indexService = indicesService.indexService("test"); + IndexShard shard = indexService.getShardOrNull(0); + client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(randomBoolean()).get(); + client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefresh(true).get(); + + try (Engine.Searcher searcher = shard.acquireSearcher("test")) { + TopDocs search = searcher.searcher().search(new TermQuery(new Term("foo", "bar")), 10); + assertEquals(search.totalHits, 1); + search = searcher.searcher().search(new TermQuery(new Term("foobar", "bar")), 10); + assertEquals(search.totalHits, 1); + } + + ShardRouting routing = new ShardRouting(shard.routingEntry()); + shard.close("simon says", true); + IndexServicesProvider indexServices = indexService.getIndexServices(); + IndexSearcherWrapper wrapper = new IndexSearcherWrapper() { + @Override + public DirectoryReader wrap(DirectoryReader reader) throws IOException { + return new FieldMaskingReader("foo", reader); + } + + @Override + public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { + return searcher; + } + }; + + IndexServicesProvider newProvider = new IndexServicesProvider(indexServices.getIndicesLifecycle(), indexServices.getThreadPool(), indexServices.getMapperService(), indexServices.getQueryParserService(), indexServices.getIndexCache(), indexServices.getIndexAliasesService(), indexServices.getIndicesQueryCache(), indexServices.getCodecService(), indexServices.getTermVectorsService(), indexServices.getIndexFieldDataService(), indexServices.getWarmer(), indexServices.getSimilarityService(), indexServices.getFactory(), indexServices.getBigArrays(), wrapper); + IndexShard newShard = new IndexShard(shard.shardId(), shard.indexSettings, shard.shardPath(), shard.store(), newProvider); + + ShardRoutingHelper.reinit(routing); + newShard.updateRoutingEntry(routing, false); + DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); + assertTrue(newShard.recoverFromStore(routing, localNode)); + routing = new ShardRouting(routing); + ShardRoutingHelper.moveToStarted(routing); + newShard.updateRoutingEntry(routing, true); + try (Engine.Searcher searcher = newShard.acquireSearcher("test")) { + TopDocs search = searcher.searcher().search(new TermQuery(new Term("foo", "bar")), 10); + assertEquals(search.totalHits, 0); + search = searcher.searcher().search(new TermQuery(new Term("foobar", "bar")), 10); + assertEquals(search.totalHits, 1); + } + newShard.close("just do it", randomBoolean()); + } + + private static class FieldMaskingReader extends FilterDirectoryReader { + + + private final String field; + + public FieldMaskingReader(String field, DirectoryReader in) throws IOException { + super(in, new SubReaderWrapper() { + private final String filteredField = field; + @Override + public LeafReader wrap(LeafReader reader) { + return new FilterLeafReader(reader) { + @Override + public Fields fields() throws IOException { + return new FilterFields(super.fields()) { + @Override + public Terms terms(String field) throws IOException { + return filteredField.equals(field) ? null : super.terms(field); + } + }; + } + }; + } + }); + this.field = field; + + } + + @Override + protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { + return new FieldMaskingReader(field, in); + } + } }