From c0f8223a6c80f05a4f5d234b2c5e5b39c95b95b1 Mon Sep 17 00:00:00 2001 From: LocJayMP Date: Thu, 25 Aug 2011 17:58:55 -0400 Subject: [PATCH] adding escape html ability --- .../action/search/SearchRequestBuilder.java | 11 +++ .../search/highlight/HighlightBuilder.java | 31 +++++++- .../search/highlight/HighlightPhase.java | 20 ++++- .../highlight/HighlighterParseElement.java | 8 ++ .../highlight/SearchContextHighlight.java | 15 +++- .../highlight/HighlighterSearchTests.java | 74 +++++++++++++++++++ 6 files changed, 151 insertions(+), 8 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java index 0e711cf8ab1..f677a8d569f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java @@ -490,6 +490,8 @@ public class SearchRequestBuilder extends BaseRequestBuilderstyled and default. @@ -118,6 +127,17 @@ public class HighlightBuilder implements ToXContent { return this; } + + /** + * Set encoder for the highlighting + * are styled and default. + * + * @param encoder name + */ + public HighlightBuilder encoder(String encoder) { + this.encoder = encoder; + return this; + } /** * Explicitly set the pre tags that will be used for highlighting. */ @@ -158,6 +178,9 @@ public class HighlightBuilder implements ToXContent { if (order != null) { builder.field("order", order); } + if (encoder != null) { + builder.field("encoder", encoder); + } if (fields != null) { builder.startObject("fields"); for (Field field : fields) { @@ -171,10 +194,12 @@ public class HighlightBuilder implements ToXContent { if (field.fragmentOffset() != -1) { builder.field("fragment_offset", field.fragmentOffset()); } + builder.endObject(); } builder.endObject(); } + builder.endObject(); return builder; } @@ -185,7 +210,6 @@ public class HighlightBuilder implements ToXContent { private int fragmentOffset = -1; private int numOfFragments = -1; - private Field(String name) { this.name = name; } @@ -220,5 +244,6 @@ public class HighlightBuilder implements ToXContent { this.numOfFragments = numOfFragments; return this; } + } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index e0df3977345..8908147f1d0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -57,7 +57,9 @@ import static org.elasticsearch.common.collect.Maps.*; */ public class HighlightPhase implements SearchHitPhase { - private static final Encoder DEFAULT_ENCODER = new DefaultEncoder(); + private Encoder encoder; + private FragListBuilder fraglistbuilder; + private FragmentsBuilder fragmentbuiler; @Override public Map parseElements() { return ImmutableMap.of("highlight", new HighlighterParseElement()); @@ -73,6 +75,10 @@ public class HighlightPhase implements SearchHitPhase { Map highlightFields = newHashMap(); for (SearchContextHighlight.Field field : context.highlight().fields()) { + if (field.encoder().equals("html")) + encoder = new SimpleHTMLEncoder(); + else + encoder = new DefaultEncoder(); FieldMapper mapper = documentMapper.mappers().smartNameFieldMapper(field.field()); if (mapper == null) { MapperService.SmartNameFieldMappers fullMapper = context.mapperService().smartName(field.field()); @@ -110,7 +116,10 @@ public class HighlightPhase implements SearchHitPhase { fragmenter = new SimpleSpanFragmenter(queryScorer, field.fragmentCharSize()); } Formatter formatter = new SimpleHTMLFormatter(field.preTags()[0], field.postTags()[0]); - Highlighter highlighter = new Highlighter(formatter, DEFAULT_ENCODER, queryScorer); + + + + Highlighter highlighter = new Highlighter(formatter, encoder, queryScorer); highlighter.setTextFragmenter(fragmenter); List textsToHighlight; @@ -186,7 +195,8 @@ public class HighlightPhase implements SearchHitPhase { try { // a HACK to make highlighter do highlighting, even though its using the single frag list builder int numberOfFragments = field.numberOfFragments() == 0 ? 1 : field.numberOfFragments(); - fragments = highlighter.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments); + fragments = highlighter.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments, + this.fraglistbuilder, this.fragmentbuiler, field.preTags(), field.postTags(), encoder); } catch (IOException e) { throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + field.field() + "]", e); } @@ -241,7 +251,9 @@ public class HighlightPhase implements SearchHitPhase { } } } - + this.fraglistbuilder = fragListBuilder; + this.fragmentbuiler = fragmentsBuilder; return new FastVectorHighlighter(true, false, fragListBuilder, fragmentsBuilder); } + } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java index 9c3f54291ec..8e72c05351c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java @@ -72,6 +72,7 @@ public class HighlighterParseElement implements SearchParseElement { boolean globalHighlightFilter = true; int globalFragmentSize = 100; int globalNumOfFragments = 5; + String globalEncoder = "default"; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -106,6 +107,9 @@ public class HighlighterParseElement implements SearchParseElement { } else if ("number_of_fragments".equals(topLevelFieldName) || "numberOfFragments".equals(topLevelFieldName)) { globalNumOfFragments = parser.intValue(); } + else if ("encoder".equals(topLevelFieldName)){ + globalEncoder = parser.text(); + } } else if (token == XContentParser.Token.START_OBJECT) { if ("fields".equals(topLevelFieldName)) { String highlightFieldName = null; @@ -176,6 +180,10 @@ public class HighlighterParseElement implements SearchParseElement { if (field.numberOfFragments() == -1) { field.numberOfFragments(globalNumOfFragments); } + if (field.encoder() == null){ + field.encoder(globalEncoder); + } + } context.highlight(new SearchContextHighlight(fields)); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java index e78ac5c0be1..7717196073a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java @@ -19,6 +19,8 @@ package org.elasticsearch.search.highlight; +import org.elasticsearch.common.io.stream.StreamInput; + import java.util.List; /** @@ -46,6 +48,8 @@ public class SearchContextHighlight { private int fragmentOffset = -1; + private String encoder; + private String[] preTags; private String[] postTags; @@ -86,7 +90,15 @@ public class SearchContextHighlight { this.fragmentOffset = fragmentOffset; } - public String[] preTags() { + public String encoder(){ + return encoder; + } + + public void encoder(String encoder){ + this.encoder = encoder; + } + + public String[] preTags() { return preTags; } @@ -117,5 +129,6 @@ public class SearchContextHighlight { public void highlightFilter(boolean highlightFilter) { this.highlightFilter = highlightFilter; } + } } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java index 4b85fe5565e..ba5ce18fddc 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java @@ -437,4 +437,78 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("hlighting bug present in elasticsearch ")); } } + + @Test public void testEscapeHtml() throws Exception { + + try { + client.admin().indices().prepareDelete("test").execute().actionGet(); + } catch (Exception e) { + // ignore + } + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2)) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("title").field("type", "string").field("store", "yes") + .endObject().endObject().endObject()) + .execute().actionGet(); + + for (int i = 0; i < 5; i++) { + client.prepareIndex("test", "type1", Integer.toString(i)) + .setSource("title", "This is a html escaping highlighting test for *&? elasticsearch").setRefresh(true).execute().actionGet(); + } + SearchSourceBuilder source = searchSource() + .query(termQuery("field1", "test")) + .from(0).size(60).explain(true) + .highlight(highlight().field("field1", 100, 0).order("score").preTags("").postTags("")); + + + SearchResponse search = client.prepareSearch() + .setQuery(fieldQuery("title", "test")).setEncoder("html") + .addHighlightedField("title",50,1,10) + .execute().actionGet(); + + + assertThat(search.hits().totalHits(), equalTo(5l)); + assertThat(search.hits().hits().length, equalTo(5)); + assertThat(search.getFailedShards(), equalTo(0)); + + for (SearchHit hit : search.hits()) { + // LUCENE 3.1 UPGRADE: Caused adding the space at the end... + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a html escaping highlighting test for *&? elasticsearch")); + } + } + @Test public void testEscapeHtml_vector() throws Exception { + + try { + client.admin().indices().prepareDelete("test").execute().actionGet(); + } catch (Exception e) { + // ignore + } + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2)) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties") + .startObject("title").field("type", "string").field("store", "yes").field("term_vector", "with_positions_offsets").endObject() + .endObject().endObject().endObject()) + .execute().actionGet(); + + for (int i = 0; i < 5; i++) { + client.prepareIndex("test", "type1", Integer.toString(i)) + .setSource("title", "This is a html escaping highlighting test for *&? elasticsearch").setRefresh(true).execute().actionGet(); + } + + SearchResponse search = client.prepareSearch() + .setQuery(fieldQuery("title", "test")).setEncoder("html") + .addHighlightedField("title",50,1,10) + .execute().actionGet(); + + + assertThat(search.hits().totalHits(), equalTo(5l)); + assertThat(search.hits().hits().length, equalTo(5)); + assertThat(search.getFailedShards(), equalTo(0)); + + for (SearchHit hit : search.hits()) { + // LUCENE 3.1 UPGRADE: Caused adding the space at the end... + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("hlighting test for *&? elasticsearch ")); + } + } }