Improve multi field mapper with highlighting based on source, closes #1559.

This commit is contained in:
Shay Banon 2011-12-22 02:24:36 +02:00
parent c0053a601b
commit fe4ba2ad55
8 changed files with 197 additions and 4 deletions

View File

@ -41,6 +41,8 @@ public class ContentPath {
private String[] path = new String[10];
private String sourcePath;
public ContentPath() {
this(0);
}
@ -58,6 +60,7 @@ public class ContentPath {
public void reset() {
this.index = 0;
this.sourcePath = null;
}
public void add(String name) {
@ -96,4 +99,14 @@ public class ContentPath {
public void pathType(Type type) {
this.pathType = type;
}
public String sourcePath(String sourcePath) {
String orig = this.sourcePath;
this.sourcePath = sourcePath;
return orig;
}
public String sourcePath() {
return this.sourcePath;
}
}

View File

@ -44,6 +44,8 @@ public interface FieldMapper<T> {
private final String fullName;
private final String sourcePath;
private final Term indexNameTermFactory;
public Names(String name) {
@ -51,10 +53,15 @@ public interface FieldMapper<T> {
}
public Names(String name, String indexName, String indexNameClean, String fullName) {
this(name, indexName, indexNameClean, fullName, fullName);
}
public Names(String name, String indexName, String indexNameClean, String fullName, @Nullable String sourcePath) {
this.name = name.intern();
this.indexName = indexName.intern();
this.indexNameClean = indexNameClean.intern();
this.fullName = fullName.intern();
this.sourcePath = sourcePath == null ? this.fullName : sourcePath.intern();
this.indexNameTermFactory = new Term(indexName, "");
}
@ -87,6 +94,13 @@ public interface FieldMapper<T> {
return fullName;
}
/**
* The dot path notation to extract the value from source.
*/
public String sourcePath() {
return sourcePath;
}
public Term createIndexNameTerm(String value) {
return indexNameTermFactory.createTerm(value);
}

View File

@ -182,7 +182,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
}
protected Names buildNames(BuilderContext context) {
return new Names(name, buildIndexName(context), indexName == null ? name : indexName, buildFullName(context));
return new Names(name, buildIndexName(context), indexName == null ? name : indexName, buildFullName(context), context.path().sourcePath());
}
protected String buildIndexName(BuilderContext context) {

View File

@ -82,6 +82,7 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll {
defaultMapper = defaultMapperBuilder.build(context);
}
String origSourcePath = context.path().sourcePath(context.path().fullPathAsText(name));
context.path().add(name);
Map<String, Mapper> mappers = new HashMap<String, Mapper>();
for (Mapper.Builder builder : mappersBuilders) {
@ -89,6 +90,7 @@ public class MultiFieldMapper implements Mapper, AllFieldMapper.IncludeInAll {
mappers.put(mapper.name(), mapper);
}
context.path().remove();
context.path().sourcePath(origSourcePath);
context.path().pathType(origPathType);

View File

@ -182,7 +182,7 @@ public class HighlightPhase implements FetchSubPhase {
SearchLookup lookup = context.lookup();
lookup.setNextReader(hitContext.reader());
lookup.setNextDocId(hitContext.docId());
textsToHighlight = lookup.source().extractRawValues(mapper.names().fullName());
textsToHighlight = lookup.source().extractRawValues(mapper.names().sourcePath());
}
// a HACK to make highlighter do highlighting, even though its using the single frag list builder

View File

@ -52,7 +52,7 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder
lookup.setNextReader(reader);
lookup.setNextDocId(docId);
List<Object> values = lookup.source().extractRawValues(mapper.names().fullName());
List<Object> values = lookup.source().extractRawValues(mapper.names().sourcePath());
Field[] fields = new Field[values.size()];
for (int i = 0; i < values.size(); i++) {
fields[i] = new Field(mapper.names().indexName(), values.get(i).toString(), Field.Store.NO, Field.Index.ANALYZED);

View File

@ -54,7 +54,7 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder {
lookup.setNextReader(reader);
lookup.setNextDocId(docId);
List<Object> values = lookup.source().extractRawValues(mapper.names().fullName());
List<Object> values = lookup.source().extractRawValues(mapper.names().sourcePath());
if (values.isEmpty()) {
return EMPTY_FIELDS;
}

View File

@ -527,4 +527,168 @@ public class HighlighterSearchTests extends AbstractNodesTests {
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting <em>test</em> for *&amp;? elasticsearch "));
}
}
@Test
public void testMultiMapperVectorWithStore() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2))
.addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
.startObject("title").field("type", "multi_field").startObject("fields")
.startObject("title").field("type", "string").field("store", "yes").field("term_vector", "with_positions_offsets").endObject()
.startObject("key").field("type", "string").field("store", "yes").field("term_vector", "with_positions_offsets").field("analyzer", "whitespace").endObject()
.endObject().endObject()
.endObject().endObject().endObject())
.execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
// simple search on body with standard analyzer with a simple field query
SearchResponse search = client.prepareSearch()
.setQuery(fieldQuery("title", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
SearchHit hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo(" is a <em>test</em> "));
// search on title.key and highlight on title
search = client.prepareSearch()
.setQuery(fieldQuery("title.key", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title.key", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em> "));
}
@Test
public void testMultiMapperVectorFromSource() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2))
.addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
.startObject("title").field("type", "multi_field").startObject("fields")
.startObject("title").field("type", "string").field("store", "no").field("term_vector", "with_positions_offsets").endObject()
.startObject("key").field("type", "string").field("store", "no").field("term_vector", "with_positions_offsets").field("analyzer", "whitespace").endObject()
.endObject().endObject()
.endObject().endObject().endObject())
.execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
// simple search on body with standard analyzer with a simple field query
SearchResponse search = client.prepareSearch()
.setQuery(fieldQuery("title", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
SearchHit hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo(" is a <em>test</em> "));
// search on title.key and highlight on title.key
search = client.prepareSearch()
.setQuery(fieldQuery("title.key", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title.key", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em> "));
}
@Test
public void testMultiMapperNoVectorWithStore() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2))
.addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
.startObject("title").field("type", "multi_field").startObject("fields")
.startObject("title").field("type", "string").field("store", "yes").field("term_vector", "no").endObject()
.startObject("key").field("type", "string").field("store", "yes").field("term_vector", "no").field("analyzer", "whitespace").endObject()
.endObject().endObject()
.endObject().endObject().endObject())
.execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
// simple search on body with standard analyzer with a simple field query
SearchResponse search = client.prepareSearch()
.setQuery(fieldQuery("title", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
SearchHit hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a <em>test</em>"));
// search on title.key and highlight on title
search = client.prepareSearch()
.setQuery(fieldQuery("title.key", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title.key", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
}
@Test
public void testMultiMapperNoVectorFromSource() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2))
.addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
.startObject("title").field("type", "multi_field").startObject("fields")
.startObject("title").field("type", "string").field("store", "no").field("term_vector", "no").endObject()
.startObject("key").field("type", "string").field("store", "no").field("term_vector", "no").field("analyzer", "whitespace").endObject()
.endObject().endObject()
.endObject().endObject().endObject())
.execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("title", "this is a test").execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
// simple search on body with standard analyzer with a simple field query
SearchResponse search = client.prepareSearch()
.setQuery(fieldQuery("title", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
SearchHit hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a <em>test</em>"));
// search on title.key and highlight on title.key
search = client.prepareSearch()
.setQuery(fieldQuery("title.key", "this is a test"))
.setHighlighterEncoder("html")
.addHighlightedField("title.key", 50, 1)
.execute().actionGet();
assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
hit = search.hits().getAt(0);
assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
}
}