From 64d5ea13d592d8c038a4321cf3b31ff96148817e Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Fri, 25 Sep 2015 10:30:30 +0200 Subject: [PATCH] Move ShardTermVectorService to be on indices level as TermVectorService There is no need to have term vectors service on the shard level where it's created for every shard. This commit moves it to a higher level which makes shard creation slightly simpler and reduces the number of long living objects. --- .../TransportShardMultiTermsVectorAction.java | 3 +- .../TransportTermVectorsAction.java | 2 +- .../elasticsearch/index/shard/IndexShard.java | 18 ++++---- .../index/shard/IndexShardModule.java | 2 - .../index/shard/ShadowIndexShard.java | 4 +- ...rsService.java => TermVectorsService.java} | 45 ++++++++----------- .../elasticsearch/indices/IndicesModule.java | 2 + .../search/fetch/FetchSubPhasePluginIT.java | 4 +- 8 files changed, 37 insertions(+), 43 deletions(-) rename core/src/main/java/org/elasticsearch/index/termvectors/{ShardTermVectorsService.java => TermVectorsService.java} (86%) diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java index 5f78038f80c..4242e8d9684 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.termvectors; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; @@ -81,7 +80,7 @@ public class TransportShardMultiTermsVectorAction extends TransportSingleShardAc try { IndexService indexService = indicesService.indexServiceSafe(request.index()); IndexShard indexShard = indexService.shardSafe(shardId.id()); - TermVectorsResponse termVectorsResponse = indexShard.termVectorsService().getTermVectors(termVectorsRequest, shardId.getIndex()); + TermVectorsResponse termVectorsResponse = indexShard.getTermVectors(termVectorsRequest); termVectorsResponse.updateTookInMillis(termVectorsRequest.startTime()); response.add(request.locations.get(i), termVectorsResponse); } catch (Throwable t) { diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java index 84487f81b43..80ec12e6843 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java @@ -83,7 +83,7 @@ public class TransportTermVectorsAction extends TransportSingleShardAction multibinder diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java index 74eb45fe792..6c45331f826 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java @@ -40,7 +40,7 @@ import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.termvectors.ShardTermVectorsService; +import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.indices.IndicesLifecycle; import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.cache.query.IndicesQueryCache; @@ -62,7 +62,7 @@ public final class ShadowIndexShard extends IndexShard { ThreadPool threadPool, MapperService mapperService, IndexQueryParserService queryParserService, IndexCache indexCache, IndexAliasesService indexAliasesService, IndicesQueryCache indicesQueryCache, - CodecService codecService, ShardTermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, + CodecService codecService, TermVectorsService termVectorsService, IndexFieldDataService indexFieldDataService, IndexService indexService, @Nullable IndicesWarmer warmer, SnapshotDeletionPolicy deletionPolicy, SimilarityService similarityService, EngineFactory factory, ClusterService clusterService, diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java similarity index 86% rename from core/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsService.java rename to core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index cb34c1167d5..5b27d327806 100644 --- a/core/src/main/java/org/elasticsearch/index/termvectors/ShardTermVectorsService.java +++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; @@ -42,10 +43,7 @@ import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.dfs.AggregatedDfs; import java.io.IOException; @@ -56,27 +54,20 @@ import static org.elasticsearch.index.mapper.SourceToParse.source; /** */ -public class ShardTermVectorsService extends AbstractIndexShardComponent { +public class TermVectorsService { - private IndexShard indexShard; private final MappingUpdatedAction mappingUpdatedAction; private final TransportDfsOnlyAction dfsAction; @Inject - public ShardTermVectorsService(ShardId shardId, @IndexSettings Settings indexSettings, MappingUpdatedAction mappingUpdatedAction, TransportDfsOnlyAction dfsAction) { - super(shardId, indexSettings); + public TermVectorsService(MappingUpdatedAction mappingUpdatedAction, TransportDfsOnlyAction dfsAction) { this.mappingUpdatedAction = mappingUpdatedAction; this.dfsAction = dfsAction; } - // sadly, to overcome cyclic dep, we need to do this and inject it ourselves... - public ShardTermVectorsService setIndexShard(IndexShard indexShard) { - this.indexShard = indexShard; - return this; - } - public TermVectorsResponse getTermVectors(TermVectorsRequest request, String concreteIndex) { - final TermVectorsResponse termVectorsResponse = new TermVectorsResponse(concreteIndex, request.type(), request.id()); + public TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequest request) { + final TermVectorsResponse termVectorsResponse = new TermVectorsResponse(indexShard.shardId().index().name(), request.type(), request.id()); final Term uidTerm = new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id())); Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), uidTerm).version(request.version()).versionType(request.versionType())); @@ -94,7 +85,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { /* handle potential wildcards in fields */ if (request.selectedFields() != null) { - handleFieldWildcards(request); + handleFieldWildcards(indexShard, request); } final Engine.Searcher searcher = indexShard.acquireSearcher("term_vector"); @@ -103,7 +94,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); /* from an artificial document */ if (request.doc() != null) { - termVectorsByField = generateTermVectorsFromDoc(request, !docFromTranslog); + termVectorsByField = generateTermVectorsFromDoc(indexShard, request, !docFromTranslog); // if no document indexed in shard, take the queried document itself for stats if (topLevelFields == null) { topLevelFields = termVectorsByField; @@ -122,7 +113,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { } // fields without term vectors if (selectedFields != null) { - termVectorsByField = addGeneratedTermVectors(get, termVectorsByField, request, selectedFields); + termVectorsByField = addGeneratedTermVectors(indexShard, get, termVectorsByField, request, selectedFields); } termVectorsResponse.setDocVersion(docIdAndVersion.version); termVectorsResponse.setExists(true); @@ -158,7 +149,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { return termVectorsResponse; } - private void handleFieldWildcards(TermVectorsRequest request) { + private void handleFieldWildcards(IndexShard indexShard, TermVectorsRequest request) { Set fieldNames = new HashSet<>(); for (String pattern : request.selectedFields()) { fieldNames.addAll(indexShard.mapperService().simpleMatchToIndexNames(pattern)); @@ -178,7 +169,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { return true; } - private Fields addGeneratedTermVectors(Engine.GetResult get, Fields termVectorsByField, TermVectorsRequest request, Set selectedFields) throws IOException { + private Fields addGeneratedTermVectors(IndexShard indexShard, Engine.GetResult get, Fields termVectorsByField, TermVectorsRequest request, Set selectedFields) throws IOException { /* only keep valid fields */ Set validFields = new HashSet<>(); for (String field : selectedFields) { @@ -201,7 +192,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { /* generate term vectors from fetched document fields */ GetResult getResult = indexShard.getService().get( get, request.id(), request.type(), validFields.toArray(Strings.EMPTY_ARRAY), null, false); - Fields generatedTermVectors = generateTermVectors(getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer(), validFields); + Fields generatedTermVectors = generateTermVectors(indexShard, getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer(), validFields); /* merge with existing Fields */ if (termVectorsByField == null) { @@ -211,7 +202,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { } } - private Analyzer getAnalyzerAtField(String field, @Nullable Map perFieldAnalyzer) { + private Analyzer getAnalyzerAtField(IndexShard indexShard, String field, @Nullable Map perFieldAnalyzer) { MapperService mapperService = indexShard.mapperService(); Analyzer analyzer; if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) { @@ -235,7 +226,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { return selectedFields; } - private Fields generateTermVectors(Collection getFields, boolean withOffsets, @Nullable Map perFieldAnalyzer, Set fields) + private Fields generateTermVectors(IndexShard indexShard, Collection getFields, boolean withOffsets, @Nullable Map perFieldAnalyzer, Set fields) throws IOException { /* store document in memory index */ MemoryIndex index = new MemoryIndex(withOffsets); @@ -245,7 +236,7 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { // some fields are returned even when not asked for, eg. _timestamp continue; } - Analyzer analyzer = getAnalyzerAtField(field, perFieldAnalyzer); + Analyzer analyzer = getAnalyzerAtField(indexShard, field, perFieldAnalyzer); for (Object text : getField.getValues()) { index.addField(field, text.toString(), analyzer); } @@ -254,9 +245,9 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { return MultiFields.getFields(index.createSearcher().getIndexReader()); } - private Fields generateTermVectorsFromDoc(TermVectorsRequest request, boolean doAllFields) throws Throwable { + private Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request, boolean doAllFields) throws Throwable { // parse the document, at the moment we do update the mapping, just like percolate - ParsedDocument parsedDocument = parseDocument(indexShard.shardId().getIndex(), request.type(), request.doc()); + ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndex(), request.type(), request.doc()); // select the right fields and generate term vectors ParseContext.Document doc = parsedDocument.rootDoc(); @@ -282,10 +273,10 @@ public class ShardTermVectorsService extends AbstractIndexShardComponent { String[] values = doc.getValues(field.name()); getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values))); } - return generateTermVectors(getFields, request.offsets(), request.perFieldAnalyzer(), seenFields); + return generateTermVectors(indexShard, getFields, request.offsets(), request.perFieldAnalyzer(), seenFields); } - private ParsedDocument parseDocument(String index, String type, BytesReference doc) throws Throwable { + private ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) throws Throwable { MapperService mapperService = indexShard.mapperService(); // TODO: make parsing not dynamically create fields not in the original mapping diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java index 32c7bc833b0..c875fa6a892 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.util.ExtensionPoint; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryParser; import org.elasticsearch.index.query.MoreLikeThisQueryParser; +import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.indices.analysis.HunspellService; import org.elasticsearch.indices.analysis.IndicesAnalysisService; import org.elasticsearch.indices.cache.query.IndicesQueryCache; @@ -147,6 +148,7 @@ public class IndicesModule extends AbstractModule { bind(UpdateHelper.class).asEagerSingleton(); bind(MetaDataIndexUpgradeService.class).asEagerSingleton(); bind(IndicesFieldDataCacheListener.class).asEagerSingleton(); + bind(TermVectorsService.class).asEagerSingleton(); } protected void bindQueryParsersExtension() { diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index e07bb73d6a6..6280f4af524 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -28,8 +28,10 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.common.Priority; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchModule; @@ -166,7 +168,7 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase { hitField = new InternalSearchHitField(NAMES[0], new ArrayList<>(1)); hitContext.hit().fields().put(NAMES[0], hitField); } - TermVectorsResponse termVector = context.indexShard().termVectorsService().getTermVectors(new TermVectorsRequest(context.indexShard().indexService().index().getName(), hitContext.hit().type(), hitContext.hit().id()), context.indexShard().indexService().index().getName()); + TermVectorsResponse termVector = context.indexShard().getTermVectors(new TermVectorsRequest(context.indexShard().indexService().index().getName(), hitContext.hit().type(), hitContext.hit().id())); try { Map tv = new HashMap<>(); TermsEnum terms = termVector.getFields().terms(field).iterator();