honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 10:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve version loading by going directly to the relevant reader

Browse Source
This commit is contained in:
kimchy 2011-01-06 11:50:16 +02:00
parent 26c5f6c482
commit b9e808f755
10 changed files with 132 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchModule.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.search.fetch.FetchPhase;
import org.elasticsearch.search.fetch.explain.ExplainSearchHitPhase;
import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersSearchHitPhase;
import org.elasticsearch.search.fetch.script.ScriptFieldsSearchHitPhase;
import org.elasticsearch.search.fetch.version.VersionSearchHitPhase;
import org.elasticsearch.search.highlight.HighlightPhase;
import org.elasticsearch.search.query.QueryPhase;
@ -52,6 +53,7 @@ public class SearchModule extends AbstractModule implements SpawnModules {
bind(FetchPhase.class).asEagerSingleton();
bind(ExplainSearchHitPhase.class).asEagerSingleton();
bind(ScriptFieldsSearchHitPhase.class).asEagerSingleton();
bind(VersionSearchHitPhase.class).asEagerSingleton();
bind(MatchedFiltersSearchHitPhase.class).asEagerSingleton();
bind(HighlightPhase.class).asEagerSingleton();

33
modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View File

@ -23,10 +23,8 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.uid.UidField;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchParseElement;
@ -34,6 +32,7 @@ import org.elasticsearch.search.SearchPhase;
import org.elasticsearch.search.fetch.explain.ExplainSearchHitPhase;
import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersSearchHitPhase;
import org.elasticsearch.search.fetch.script.ScriptFieldsSearchHitPhase;
import org.elasticsearch.search.fetch.version.VersionSearchHitPhase;
import org.elasticsearch.search.highlight.HighlightPhase;
import org.elasticsearch.search.internal.InternalSearchHit;
import org.elasticsearch.search.internal.InternalSearchHitField;
@ -53,8 +52,8 @@ public class FetchPhase implements SearchPhase {
private final SearchHitPhase[] hitPhases;
@Inject public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsSearchHitPhase scriptFieldsPhase,
MatchedFiltersSearchHitPhase matchFiltersPhase, ExplainSearchHitPhase explainPhase) {
this.hitPhases = new SearchHitPhase[]{scriptFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase};
MatchedFiltersSearchHitPhase matchFiltersPhase, ExplainSearchHitPhase explainPhase, VersionSearchHitPhase versionPhase) {
this.hitPhases = new SearchHitPhase[]{scriptFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase};
}
@Override public Map<String, ? extends SearchParseElement> parseElements() {
@ -83,12 +82,8 @@ public class FetchPhase implements SearchPhase {
byte[] source = extractSource(doc, documentMapper);
// get the version
long version = UidField.loadVersion(context.searcher().getIndexReader(), new Term(UidFieldMapper.NAME, doc.get(UidFieldMapper.NAME)));
if (version < 0) {
version = -1;
}
InternalSearchHit searchHit = new InternalSearchHit(docId, uid.id(), uid.type(), version, source, null);
InternalSearchHit searchHit = new InternalSearchHit(docId, uid.id(), uid.type(), source, null);
hits[index] = searchHit;
for (Object oField : doc.getFields()) {
@ -134,22 +129,14 @@ public class FetchPhase implements SearchPhase {
hitField.values().add(value);
}
boolean hitPhaseExecutionRequired = false;
int readerIndex = context.searcher().readerIndex(docId);
IndexReader subReader = context.searcher().subReaders()[readerIndex];
int subDoc = docId - context.searcher().docStarts()[readerIndex];
for (SearchHitPhase hitPhase : hitPhases) {
SearchHitPhase.HitContext hitContext = new SearchHitPhase.HitContext();
if (hitPhase.executionNeeded(context)) {
hitPhaseExecutionRequired = true;
break;
}
}
if (hitPhaseExecutionRequired) {
int readerIndex = context.searcher().readerIndex(docId);
IndexReader subReader = context.searcher().subReaders()[readerIndex];
int subDoc = docId - context.searcher().docStarts()[readerIndex];
for (SearchHitPhase hitPhase : hitPhases) {
if (hitPhase.executionNeeded(context)) {
hitPhase.execute(context, searchHit, uid, subReader, subDoc);
}
hitContext.reset(searchHit, subReader, subDoc, doc);
hitPhase.execute(context, hitContext);
}
}
}

33
modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/SearchHitPhase.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.fetch;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.index.mapper.Uid;
@ -33,6 +34,36 @@ import java.util.Map;
*/
public interface SearchHitPhase {
public static class HitContext {
private InternalSearchHit hit;
private IndexReader reader;
private int docId;
private Document doc;
public void reset(InternalSearchHit hit, IndexReader reader, int docId, Document doc) {
this.hit = hit;
this.reader = reader;
this.docId = docId;
this.doc = doc;
}
public InternalSearchHit hit() {
return hit;
}
public IndexReader reader() {
return reader;
}
public int docId() {
return docId;
}
public Document doc() {
return doc;
}
}
Map<String, ? extends SearchParseElement> parseElements();
boolean executionNeeded(SearchContext context);
@ -40,5 +71,5 @@ public interface SearchHitPhase {
/**
* Executes the hit level phase, with a reader and doc id (note, its a low level reader, and the matching doc).
*/
void execute(SearchContext context, InternalSearchHit hit, Uid uid, IndexReader reader, int docId) throws ElasticSearchException;
void execute(SearchContext context, HitContext hitContext) throws ElasticSearchException;
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/explain/ExplainSearchHitPhase.java
View File

@ -45,12 +45,12 @@ public class ExplainSearchHitPhase implements SearchHitPhase {
return context.explain();
}
@Override public void execute(SearchContext context, InternalSearchHit hit, Uid uid, IndexReader reader, int docId) throws ElasticSearchException {
@Override public void execute(SearchContext context, HitContext hitContext) throws ElasticSearchException {
try {
// we use the top level doc id, since we work with the top level searcher
hit.explanation(context.searcher().explain(context.query(), hit.docId()));
hitContext.hit().explanation(context.searcher().explain(context.query(), hitContext.hit().docId()));
} catch (IOException e) {
throw new FetchPhaseExecutionException(context, "Failed to explain doc [" + docId + "]", e);
throw new FetchPhaseExecutionException(context, "Failed to explain doc [" + hitContext.hit().type() + "#" + hitContext.hit().id() + "]", e);
}
}
}

8
modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/matchedfilters/MatchedFiltersSearchHitPhase.java
View File

@ -49,21 +49,21 @@ public class MatchedFiltersSearchHitPhase implements SearchHitPhase {
return !context.parsedQuery().namedFilters().isEmpty();
}
@Override public void execute(SearchContext context, InternalSearchHit hit, Uid uid, IndexReader reader, int docId) throws ElasticSearchException {
@Override public void execute(SearchContext context, HitContext hitContext) throws ElasticSearchException {
List<String> matchedFilters = Lists.newArrayListWithCapacity(2);
for (Map.Entry<String, Filter> entry : context.parsedQuery().namedFilters().entrySet()) {
String name = entry.getKey();
Filter filter = entry.getValue();
filter = context.filterCache().cache(filter);
try {
DocIdSet docIdSet = filter.getDocIdSet(reader);
if (docIdSet instanceof DocSet && ((DocSet) docIdSet).get(docId)) {
DocIdSet docIdSet = filter.getDocIdSet(hitContext.reader());
if (docIdSet instanceof DocSet && ((DocSet) docIdSet).get(hitContext.docId())) {
matchedFilters.add(name);
}
} catch (IOException e) {
// ignore
}
}
hit.matchedFilters(matchedFilters.toArray(new String[matchedFilters.size()]));
hitContext.hit().matchedFilters(matchedFilters.toArray(new String[matchedFilters.size()]));
}
}

14
modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsSearchHitPhase.java
View File

@ -54,13 +54,13 @@ public class ScriptFieldsSearchHitPhase implements SearchHitPhase {
return context.hasScriptFields();
}
@Override public void execute(SearchContext context, InternalSearchHit hit, Uid uid, IndexReader reader, int docId) throws ElasticSearchException {
@Override public void execute(SearchContext context, HitContext hitContext) throws ElasticSearchException {
for (ScriptFieldsContext.ScriptField scriptField : context.scriptFields().fields()) {
scriptField.script().setNextReader(reader);
scriptField.script().setNextReader(hitContext.reader());
Object value;
try {
value = scriptField.script().execute(docId);
value = scriptField.script().execute(hitContext.docId());
} catch (RuntimeException e) {
if (scriptField.ignoreException()) {
continue;
@ -68,14 +68,14 @@ public class ScriptFieldsSearchHitPhase implements SearchHitPhase {
throw e;
}
if (hit.fieldsOrNull() == null) {
hit.fields(new HashMap<String, SearchHitField>(2));
if (hitContext.hit().fieldsOrNull() == null) {
hitContext.hit().fields(new HashMap<String, SearchHitField>(2));
}
SearchHitField hitField = hit.fields().get(scriptField.name());
SearchHitField hitField = hitContext.hit().fields().get(scriptField.name());
if (hitField == null) {
hitField = new InternalSearchHitField(scriptField.name(), new ArrayList<Object>(2));
hit.fields().put(scriptField.name(), hitField);
hitContext.hit().fields().put(scriptField.name(), hitField);
}
hitField.values().add(value);
}

53
modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/version/VersionSearchHitPhase.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.fetch.version;
import org.apache.lucene.index.Term;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.lucene.uid.UidField;
import org.elasticsearch.index.mapper.UidFieldMapper;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.fetch.SearchHitPhase;
import org.elasticsearch.search.internal.SearchContext;
import java.util.Map;
/**
* @author kimchy (shay.banon)
*/
public class VersionSearchHitPhase implements SearchHitPhase {
@Override public Map<String, ? extends SearchParseElement> parseElements() {
return ImmutableMap.of();
}
@Override public boolean executionNeeded(SearchContext context) {
return true;
}
@Override public void execute(SearchContext context, HitContext hitContext) throws ElasticSearchException {
long version = UidField.loadVersion(hitContext.reader(), new Term(UidFieldMapper.NAME, hitContext.doc().get(UidFieldMapper.NAME)));
if (version < 0) {
version = -1;
}
hitContext.hit().version(version);
}
}

18
modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
View File

@ -66,9 +66,9 @@ public class HighlightPhase implements SearchHitPhase {
return context.highlight() != null;
}
@Override public void execute(SearchContext context, InternalSearchHit hit, Uid uid, IndexReader reader, int docId) throws ElasticSearchException {
@Override public void execute(SearchContext context, HitContext hitContext) throws ElasticSearchException {
try {
DocumentMapper documentMapper = context.mapperService().documentMapper(hit.type());
DocumentMapper documentMapper = context.mapperService().documentMapper(hitContext.hit().type());
Map<String, HighlightField> highlightFields = newHashMap();
for (SearchContextHighlight.Field field : context.highlight().fields()) {
@ -104,7 +104,7 @@ public class HighlightPhase implements SearchHitPhase {
List<Object> textsToHighlight;
if (mapper.stored()) {
try {
Document doc = reader.document(docId, new SingleFieldSelector(mapper.names().indexName()));
Document doc = hitContext.reader().document(hitContext.docId(), new SingleFieldSelector(mapper.names().indexName()));
textsToHighlight = new ArrayList<Object>(doc.getFields().size());
for (Fieldable docField : doc.getFields()) {
if (docField.stringValue() != null) {
@ -116,8 +116,8 @@ public class HighlightPhase implements SearchHitPhase {
}
} else {
SearchLookup lookup = context.lookup();
lookup.setNextReader(reader);
lookup.setNextDocId(docId);
lookup.setNextReader(hitContext.reader());
lookup.setNextDocId(hitContext.docId());
textsToHighlight = lookup.source().getValues(mapper.names().fullName());
}
@ -125,7 +125,7 @@ public class HighlightPhase implements SearchHitPhase {
try {
for (Object textToHighlight : textsToHighlight) {
String text = textToHighlight.toString();
Analyzer analyzer = context.mapperService().documentMapper(hit.type()).mappers().indexAnalyzer();
Analyzer analyzer = context.mapperService().documentMapper(hitContext.hit().type()).mappers().indexAnalyzer();
TokenStream tokenStream = analyzer.reusableTokenStream(mapper.names().indexName(), new FastStringReader(text));
TextFragment[] bestTextFragments = highlighter.getBestTextFragments(tokenStream, text, false, field.numberOfFragments());
Collections.addAll(fragsList, bestTextFragments);
@ -149,13 +149,13 @@ public class HighlightPhase implements SearchHitPhase {
highlightFields.put(highlightField.name(), highlightField);
} else {
FastVectorHighlighter highlighter = buildHighlighter(context, mapper, field);
FieldQuery fieldQuery = buildFieldQuery(highlighter, context.query(), reader, field);
FieldQuery fieldQuery = buildFieldQuery(highlighter, context.query(), hitContext.reader(), field);
String[] fragments;
try {
// a HACK to make highlighter do highlighting, even though its using the single frag list builder
int numberOfFragments = field.numberOfFragments() == 0 ? 1 : field.numberOfFragments();
fragments = highlighter.getBestFragments(fieldQuery, reader, docId, mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments);
fragments = highlighter.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments);
} catch (IOException e) {
throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + field.field() + "]", e);
}
@ -164,7 +164,7 @@ public class HighlightPhase implements SearchHitPhase {
}
}
hit.highlightFields(highlightFields);
hitContext.hit().highlightFields(highlightFields);
} finally {
CustomFieldQuery.reader.remove();
CustomFieldQuery.highlightFilters.remove();

7
modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
View File

@ -84,11 +84,10 @@ public class InternalSearchHit implements SearchHit {
}
public InternalSearchHit(int docId, String id, String type, long version, byte[] source, Map<String, SearchHitField> fields) {
public InternalSearchHit(int docId, String id, String type, byte[] source, Map<String, SearchHitField> fields) {
this.docId = docId;
this.id = id;
this.type = type;
this.version = version;
this.source = source;
this.fields = fields;
}
@ -113,6 +112,10 @@ public class InternalSearchHit implements SearchHit {
return score();
}
public void version(long version) {
this.version = version;
}
@Override public long version() {
return this.version;
}

10
modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java
View File

@ -106,9 +106,13 @@ public class SimpleVersioningTests extends AbstractNodesTests {
}
client.admin().indices().prepareRefresh().execute().actionGet();
assertThat(client.prepareGet("test", "type", "1").execute().actionGet().version(), equalTo(2l));
for (int i = 0; i < 10; i++) {
assertThat(client.prepareGet("test", "type", "1").execute().actionGet().version(), equalTo(2l));
}
SearchResponse searchResponse = client.prepareSearch().setQuery(matchAllQuery()).execute().actionGet();
assertThat(searchResponse.hits().getAt(0).version(), equalTo(2l));
for (int i = 0; i < 10; i++) {
SearchResponse searchResponse = client.prepareSearch().setQuery(matchAllQuery()).execute().actionGet();
assertThat(searchResponse.hits().getAt(0).version(), equalTo(2l));
}
}
}