Introduce Text abstraction, allowing for improved representation of strings, apply to HighlightedField (breaks backward for Java API from String to Text), closes #2093.

By introducing the Text abstraction, we can keep (long) text fields in their UTF8 bytes format, and no need to convert them to a string when serializing it back to Json for example.

The first place we can apply this is to highlighted text, which can be long.. . This does breaks backward comp. for people using the Java API where the HighlightField now has a Text as its content, and not String.

src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java

@@ -1,6 +1,7 @@
 package org.elasticsearch.common.io.stream;
 
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
 
 import java.io.IOException;
 
@@ -94,6 +95,11 @@ public abstract class AdapterStreamInput extends StreamInput {
         return in.readString();
     }
 
+    @Override
+    public Text readText() throws IOException {
+        return in.readText();
+    }
+
     @Override
     public int read(byte[] b) throws IOException {
         return in.read(b);

src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java

@@ -21,6 +21,7 @@ package org.elasticsearch.common.io.stream;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
 
 import java.io.IOException;
 
@@ -112,6 +113,11 @@ public class AdapterStreamOutput extends StreamOutput {
         out.writeString(str);
     }
 
+    @Override
+    public void writeText(Text text) throws IOException {
+        out.writeText(text);
+    }
+
     @Override
     public void writeFloat(float v) throws IOException {
         out.writeFloat(v);

src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

@@ -23,6 +23,8 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.StringAndBytesText;
+import org.elasticsearch.common.text.Text;
 import org.joda.time.DateTime;
 
 import java.io.IOException;
@@ -155,6 +157,11 @@ public abstract class StreamInput extends InputStream {
         return null;
     }
 
+    public Text readText() throws IOException {
+        // use StringAndBytes so we can cache the string if its ever converted to it
+        return new StringAndBytesText(readBytesReference());
+    }
+
     @Nullable
     public String readOptionalString() throws IOException {
         if (readBoolean()) {

src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -21,6 +21,7 @@ package org.elasticsearch.common.io.stream;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
 import org.joda.time.ReadableInstant;
 
 import java.io.IOException;
@@ -148,6 +149,12 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
+    public void writeText(Text text) throws IOException {
+        // always write the bytes...
+        // TODO: TextBytesOptimization we could potentially optimize this, and write the bytes directly to the output stream converting to UTF8 in case its a string
+        writeBytesReference(text.bytes());
+    }
+
     public void writeString(String str) throws IOException {
         int charCount = str.length();
         writeVInt(charCount);

src/main/java/org/elasticsearch/common/text/BytesText.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.text;
+
+import com.google.common.base.Charsets;
+import org.elasticsearch.common.bytes.BytesReference;
+
+/**
+ * A {@link BytesReference} representation of the text, will always convert on the fly to a {@link String}.
+ */
+public class BytesText implements Text {
+
+    private BytesReference bytes;
+
+    public BytesText(BytesReference bytes) {
+        this.bytes = bytes;
+    }
+
+    @Override
+    public boolean hasBytes() {
+        return true;
+    }
+
+    @Override
+    public BytesReference bytes() {
+        return bytes;
+    }
+
+    @Override
+    public boolean hasString() {
+        return false;
+    }
+
+    @Override
+    public String string() {
+        // TODO: we can optimize the conversion based on the bytes reference API similar to UnicodeUtil
+        if (!bytes.hasArray()) {
+            bytes = bytes.toBytesArray();
+        }
+        return new String(bytes.array(), bytes.arrayOffset(), bytes.length(), Charsets.UTF_8);
+    }
+
+    @Override
+    public String toString() {
+        return string();
+    }
+}

src/main/java/org/elasticsearch/common/text/StringAndBytesText.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.text;
+
+import com.google.common.base.Charsets;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+
+/**
+ * Both {@link String} and {@link BytesReference} representation of the text. Starts with one of those, and if
+ * the other is requests, caches the other one in a local reference so no additional conversion will be needed.
+ */
+public class StringAndBytesText implements Text {
+
+    public static final Text[] EMPTY_ARRAY = new Text[0];
+
+    public static Text[] convertFromStringArray(String[] strings) {
+        if (strings.length == 0) {
+            return EMPTY_ARRAY;
+        }
+        Text[] texts = new Text[strings.length];
+        for (int i = 0; i < strings.length; i++) {
+            texts[i] = new StringAndBytesText(strings[i]);
+        }
+        return texts;
+    }
+
+    private BytesReference bytes;
+    private String text;
+
+    public StringAndBytesText(BytesReference bytes) {
+        this.bytes = bytes;
+    }
+
+    public StringAndBytesText(String text) {
+        this.text = text;
+    }
+
+    @Override
+    public boolean hasBytes() {
+        return bytes != null;
+    }
+
+    @Override
+    public BytesReference bytes() {
+        if (bytes == null) {
+            bytes = new BytesArray(text.getBytes(Charsets.UTF_8));
+        }
+        return bytes;
+    }
+
+    @Override
+    public boolean hasString() {
+        return text != null;
+    }
+
+    @Override
+    public String string() {
+        if (text == null) {
+            if (!bytes.hasArray()) {
+                bytes = bytes.toBytesArray();
+            }
+            text = new String(bytes.array(), bytes.arrayOffset(), bytes.length());
+        }
+        return text;
+    }
+
+    @Override
+    public String toString() {
+        return string();
+    }
+}

src/main/java/org/elasticsearch/common/text/StringText.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.text;
+
+import com.google.common.base.Charsets;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+
+/**
+ * A {@link String} only representation of the text. Will always convert to bytes on the fly.
+ */
+public class StringText implements Text {
+
+    public static final Text[] EMPTY_ARRAY = new Text[0];
+
+    public static Text[] convertFromStringArray(String[] strings) {
+        if (strings.length == 0) {
+            return EMPTY_ARRAY;
+        }
+        Text[] texts = new Text[strings.length];
+        for (int i = 0; i < strings.length; i++) {
+            texts[i] = new StringText(strings[i]);
+        }
+        return texts;
+    }
+
+    private final String text;
+
+    public StringText(String text) {
+        this.text = text;
+    }
+
+    @Override
+    public boolean hasBytes() {
+        return false;
+    }
+
+    @Override
+    public BytesReference bytes() {
+        return new BytesArray(text.getBytes(Charsets.UTF_8));
+    }
+
+    @Override
+    public boolean hasString() {
+        return true;
+    }
+
+    @Override
+    public String string() {
+        return text;
+    }
+
+    @Override
+    public String toString() {
+        return string();
+    }
+}

src/main/java/org/elasticsearch/common/text/Text.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.text;
+
+import org.elasticsearch.common.bytes.BytesReference;
+
+/**
+ * Text represents a (usually) long text data. We use this abstraction instead of {@link String}
+ * so we can represent it in a more optimized manner in memory as well as serializing it over the
+ * network as well as converting it to json format.
+ */
+public interface Text {
+
+    /**
+     * Are bytes available without the need to be converted into bytes when calling {@link #bytes()}.
+     */
+    boolean hasBytes();
+
+    /**
+     * The UTF8 bytes representing the the text, might be converted on the fly, see {@link #hasBytes()}
+     */
+    BytesReference bytes();
+
+    /**
+     * Is there a {@link String} representation of the text. If not, then it {@link #hasBytes()}.
+     */
+    boolean hasString();
+
+    /**
+     * Returns the string representation of the text, might be converted to a string on the fly.
+     */
+    String string();
+
+    /**
+     * Returns the string representation of the text, might be converted to a string on the fly.
+     */
+    String toString();
+}

src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.BytesStream;
 import org.elasticsearch.common.io.FastByteArrayOutputStream;
+import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.support.XContentMapConverter;
 import org.joda.time.DateTimeZone;
 import org.joda.time.ReadableInstant;
@@ -478,6 +479,22 @@ public final class XContentBuilder implements BytesStream {
         return this;
     }
 
+    public XContentBuilder field(String name, Text value) throws IOException {
+        field(name);
+        if (value.hasBytes() && value.bytes().hasArray()) {
+            generator.writeUTF8String(value.bytes().array(), value.bytes().arrayOffset(), value.bytes().length());
+            return this;
+        }
+        if (value.hasString()) {
+            generator.writeString(value.string());
+            return this;
+        }
+        // TODO: TextBytesOptimization we can use a buffer here to convert it? maybe add a request to jackson to support InputStream as well?
+        BytesArray bytesArray = value.bytes().toBytesArray();
+        generator.writeUTF8String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length());
+        return this;
+    }
+
 
     public XContentBuilder field(String name, byte[] value, int offset, int length) throws IOException {
         field(name);
@@ -720,6 +737,8 @@ public final class XContentBuilder implements BytesStream {
             field(name, (double[]) value);
         } else if (value instanceof BytesReference) {
             field(name, (BytesReference) value);
+        } else if (value instanceof Text) {
+            field(name, (Text) value);
         } else {
             field(name, value.toString());
         }
@@ -755,6 +774,8 @@ public final class XContentBuilder implements BytesStream {
             value((ReadableInstant) value);
         } else if (value instanceof BytesReference) {
             value((BytesReference) value);
+        } else if (value instanceof Text) {
+            value((Text) value);
         } else if (value instanceof Map) {
             //noinspection unchecked
             value((Map<String, Object>) value);
@@ -986,6 +1007,23 @@ public final class XContentBuilder implements BytesStream {
         return this;
     }
 
+    public XContentBuilder value(Text value) throws IOException {
+        if (value == null) {
+            return nullValue();
+        }
+        if (value.hasBytes() && value.bytes().hasArray()) {
+            generator.writeUTF8String(value.bytes().array(), value.bytes().arrayOffset(), value.bytes().length());
+            return this;
+        }
+        if (value.hasString()) {
+            generator.writeString(value.string());
+            return this;
+        }
+        BytesArray bytesArray = value.bytes().toBytesArray();
+        generator.writeUTF8String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length());
+        return this;
+    }
+
     public XContentBuilder map(Map<String, Object> map) throws IOException {
         if (map == null) {
             return nullValue();

src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java

@@ -50,6 +50,8 @@ public interface XContentGenerator {
 
     void writeString(char[] text, int offset, int len) throws IOException;
 
+    void writeUTF8String(byte[] text, int offset, int length) throws IOException;
+
     void writeBinary(byte[] data, int offset, int len) throws IOException;
 
     void writeBinary(byte[] data) throws IOException;

src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java

@@ -89,6 +89,11 @@ public class JsonXContentGenerator implements XContentGenerator {
         generator.writeString(text, offset, len);
     }
 
+    @Override
+    public void writeUTF8String(byte[] text, int offset, int length) throws IOException {
+        generator.writeUTF8String(text, offset, length);
+    }
+
     @Override
     public void writeBinary(byte[] data, int offset, int len) throws IOException {
         generator.writeBinary(data, offset, len);

src/main/java/org/elasticsearch/search/highlight/HighlightField.java

@@ -19,29 +19,28 @@
 
 package org.elasticsearch.search.highlight;
 
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.text.StringText;
+import org.elasticsearch.common.text.Text;
 
 import java.io.IOException;
 import java.util.Arrays;
 
 /**
  * A field highlighted with its highlighted fragments.
- *
- *
  */
 public class HighlightField implements Streamable {
 
     private String name;
 
-    private String[] fragments;
+    private Text[] fragments;
 
     HighlightField() {
     }
 
-    public HighlightField(String name, String[] fragments) {
+    public HighlightField(String name, Text[] fragments) {
         this.name = name;
         this.fragments = fragments;
     }
@@ -63,14 +62,14 @@ public class HighlightField implements Streamable {
     /**
      * The highlighted fragments. <tt>null</tt> if failed to highlight (for example, the field is not stored).
      */
-    public String[] fragments() {
+    public Text[] fragments() {
         return fragments;
     }
 
     /**
      * The highlighted fragments. <tt>null</tt> if failed to highlight (for example, the field is not stored).
      */
-    public String[] getFragments() {
+    public Text[] getFragments() {
         return fragments();
     }
 
@@ -91,11 +90,11 @@ public class HighlightField implements Streamable {
         if (in.readBoolean()) {
             int size = in.readVInt();
             if (size == 0) {
-                fragments = Strings.EMPTY_ARRAY;
+                fragments = StringText.EMPTY_ARRAY;
             } else {
-                fragments = new String[size];
+                fragments = new Text[size];
                 for (int i = 0; i < size; i++) {
-                    fragments[i] = in.readUTF();
+                    fragments[i] = in.readText();
                 }
             }
         }
@@ -109,8 +108,8 @@ public class HighlightField implements Streamable {
         } else {
             out.writeBoolean(true);
             out.writeVInt(fragments.length);
-            for (String fragment : fragments) {
-                out.writeUTF(fragment);
+            for (Text fragment : fragments) {
+                out.writeText(fragment);
             }
         }
     }

src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java

@@ -41,6 +41,7 @@ import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.vectorhighlight.SimpleBoundaryScanner2;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.text.StringText;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -245,7 +246,7 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
                 }
 
                 if (fragments != null && fragments.length > 0) {
-                    HighlightField highlightField = new HighlightField(field.field(), fragments);
+                    HighlightField highlightField = new HighlightField(field.field(), StringText.convertFromStringArray(fragments));
                     highlightFields.put(highlightField.name(), highlightField);
                 }
             } else {
@@ -325,7 +326,7 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
                             entry.fragListBuilder, entry.fragmentsBuilder, field.preTags(), field.postTags(), encoder);
 
                     if (fragments != null && fragments.length > 0) {
-                        HighlightField highlightField = new HighlightField(field.field(), fragments);
+                        HighlightField highlightField = new HighlightField(field.field(), StringText.convertFromStringArray(fragments));
                         highlightFields.put(highlightField.name(), highlightField);
                     }
                 } catch (Exception e) {

src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java

@@ -29,6 +29,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -412,7 +413,7 @@ public class InternalSearchHit implements SearchHit {
                     builder.nullValue();
                 } else {
                     builder.startArray();
-                    for (String fragment : field.fragments()) {
+                    for (Text fragment : field.fragments()) {
                         builder.value(fragment);
                     }
                     builder.endArray();

src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java

@@ -108,7 +108,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(search.hits().hits().length, equalTo(5));
 
         for (SearchHit hit : search.hits()) {
-            assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
         }
 
         search = client.prepareSearch()
@@ -122,8 +122,8 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(search.hits().hits().length, equalTo(5));
 
         for (SearchHit hit : search.hits()) {
-            assertThat(hit.highlightFields().get("attachments.body").fragments()[0], equalTo("<em>attachment</em> 1"));
-            assertThat(hit.highlightFields().get("attachments.body").fragments()[1], equalTo("<em>attachment</em> 2"));
+            assertThat(hit.highlightFields().get("attachments.body").fragments()[0].string(), equalTo("<em>attachment</em> 1"));
+            assertThat(hit.highlightFields().get("attachments.body").fragments()[1].string(), equalTo("<em>attachment</em> 2"));
         }
     }
 
@@ -163,7 +163,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(search.hits().hits().length, equalTo(5));
 
         for (SearchHit hit : search.hits()) {
-            assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
         }
 
         search = client.prepareSearch()
@@ -177,79 +177,79 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(search.hits().hits().length, equalTo(5));
 
         for (SearchHit hit : search.hits()) {
-            assertThat(hit.highlightFields().get("attachments.body").fragments()[0], equalTo("<em>attachment</em> 1"));
-            assertThat(hit.highlightFields().get("attachments.body").fragments()[1], equalTo("<em>attachment</em> 2"));
+            assertThat(hit.highlightFields().get("attachments.body").fragments()[0].string(), equalTo("<em>attachment</em> 1"));
+            assertThat(hit.highlightFields().get("attachments.body").fragments()[1].string(), equalTo("<em>attachment</em> 2"));
         }
     }
 
-  @Test
-  public void testHighlightIssue1994() throws Exception {
-    try {
-      client.admin().indices().prepareDelete("test").execute().actionGet();
-    } catch (Exception e) {
-      // ignore
-    }
+    @Test
+    public void testHighlightIssue1994() throws Exception {
+        try {
+            client.admin().indices().prepareDelete("test").execute().actionGet();
+        } catch (Exception e) {
+            // ignore
+        }
 
-    client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2))
-            .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
-                    // we don't store title, now lets see if it works...
-                    .startObject("title").field("type", "string").field("store", "no").endObject()
-                    .startObject("titleTV").field("type", "string").field("store", "no").field("term_vector", "with_positions_offsets").endObject()
-                    .endObject().endObject().endObject())
-            .execute().actionGet();
+        client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 2))
+                .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
+                        // we don't store title, now lets see if it works...
+                        .startObject("title").field("type", "string").field("store", "no").endObject()
+                        .startObject("titleTV").field("type", "string").field("store", "no").field("term_vector", "with_positions_offsets").endObject()
+                        .endObject().endObject().endObject())
+                .execute().actionGet();
 
 
-    client.prepareIndex("test", "type1", "1")
-            .setSource(XContentFactory.jsonBuilder().startObject()
-                    .startArray("title")
-                      .value("This is a test on the highlighting bug present in elasticsearch")
-                      .value("The bug is bugging us")
-                    .endArray()
-                    .startArray("titleTV")
-                      .value("This is a test on the highlighting bug present in elasticsearch")
-                      .value("The bug is bugging us")
-                    .endArray()
-                    .endObject())
-            .setRefresh(true).execute().actionGet();
+        client.prepareIndex("test", "type1", "1")
+                .setSource(XContentFactory.jsonBuilder().startObject()
+                        .startArray("title")
+                        .value("This is a test on the highlighting bug present in elasticsearch")
+                        .value("The bug is bugging us")
+                        .endArray()
+                        .startArray("titleTV")
+                        .value("This is a test on the highlighting bug present in elasticsearch")
+                        .value("The bug is bugging us")
+                        .endArray()
+                        .endObject())
+                .setRefresh(true).execute().actionGet();
 
 
-    client.prepareIndex("test", "type1", "2")
-            .setSource(XContentFactory.jsonBuilder().startObject()
-                    .startArray("titleTV")
-                      .value("some text to highlight")
-                      .value("highlight other text")
-                    .endArray()
-                    .endObject())
-            .setRefresh(true).execute().actionGet();
+        client.prepareIndex("test", "type1", "2")
+                .setSource(XContentFactory.jsonBuilder().startObject()
+                        .startArray("titleTV")
+                        .value("some text to highlight")
+                        .value("highlight other text")
+                        .endArray()
+                        .endObject())
+                .setRefresh(true).execute().actionGet();
 
-    SearchResponse search = client.prepareSearch()
-            .setQuery(fieldQuery("title", "bug"))
-            .addHighlightedField("title", -1, 2)
-            .addHighlightedField("titleTV", -1, 2)
-            .execute().actionGet();
+        SearchResponse search = client.prepareSearch()
+                .setQuery(fieldQuery("title", "bug"))
+                .addHighlightedField("title", -1, 2)
+                .addHighlightedField("titleTV", -1, 2)
+                .execute().actionGet();
 
-    assertThat(search.hits().totalHits(), equalTo(1l));
-    assertThat(search.hits().hits().length, equalTo(1));
+        assertThat(search.hits().totalHits(), equalTo(1l));
+        assertThat(search.hits().hits().length, equalTo(1));
 
-    assertThat(search.hits().hits()[0].highlightFields().get("title").fragments().length, equalTo(2));
-    assertThat(search.hits().hits()[0].highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
-    assertThat(search.hits().hits()[0].highlightFields().get("title").fragments()[1], equalTo("The <em>bug</em> is bugging us"));
-    assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments().length, equalTo(2));
+        assertThat(search.hits().hits()[0].highlightFields().get("title").fragments().length, equalTo(2));
+        assertThat(search.hits().hits()[0].highlightFields().get("title").fragments()[0].string(), equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+        assertThat(search.hits().hits()[0].highlightFields().get("title").fragments()[1].string(), equalTo("The <em>bug</em> is bugging us"));
+        assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments().length, equalTo(2));
 //    assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[0], equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
-    assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[0], equalTo("highlighting <em>bug</em> present in elasticsearch")); // FastVectorHighlighter starts highlighting from startOffset - margin
-    assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[1], equalTo("The <em>bug</em> is bugging us"));
+        assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[0].string(), equalTo("highlighting <em>bug</em> present in elasticsearch")); // FastVectorHighlighter starts highlighting from startOffset - margin
+        assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[1].string(), equalTo("The <em>bug</em> is bugging us"));
 
-    search = client.prepareSearch()
-            .setQuery(fieldQuery("titleTV", "highlight"))
-            .addHighlightedField("titleTV", -1, 2)
-          . execute().actionGet();
+        search = client.prepareSearch()
+                .setQuery(fieldQuery("titleTV", "highlight"))
+                .addHighlightedField("titleTV", -1, 2)
+                .execute().actionGet();
 
-    assertThat(search.hits().totalHits(), equalTo(1l));
-    assertThat(search.hits().hits().length, equalTo(1));
-    assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments().length, equalTo(2));
-    assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[0], equalTo("text to <em>highlight</em>"));
-    assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[1], equalTo("<em>highlight</em> other text"));
-  }
+        assertThat(search.hits().totalHits(), equalTo(1l));
+        assertThat(search.hits().hits().length, equalTo(1));
+        assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments().length, equalTo(2));
+        assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[0].string(), equalTo("text to <em>highlight</em>"));
+        assertThat(search.hits().hits()[0].highlightFields().get("titleTV").fragments()[1].string(), equalTo("<em>highlight</em> other text"));
+    }
 
     @Test
     public void testPlainHighlighter() throws Exception {
@@ -275,7 +275,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a <xxx>test</xxx>"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0].string(), equalTo("this is a <xxx>test</xxx>"));
 
         logger.info("--> searching on _all, highlighting on field1");
         source = searchSource()
@@ -287,7 +287,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a <xxx>test</xxx>"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0].string(), equalTo("this is a <xxx>test</xxx>"));
 
         logger.info("--> searching on _all, highlighting on field2");
         source = searchSource()
@@ -299,7 +299,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0].string(), equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
 
         logger.info("--> searching on _all, highlighting on field2");
         source = searchSource()
@@ -311,7 +311,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0].string(), equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
     }
 
     @Test
@@ -338,7 +338,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a <xxx>test</xxx>"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0].string(), equalTo("this is a <xxx>test</xxx>"));
 
         logger.info("--> searching on _all, highlighting on field1");
         source = searchSource()
@@ -351,7 +351,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
         // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a <xxx>test</xxx>"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0].string(), equalTo("this is a <xxx>test</xxx>"));
 
         logger.info("--> searching on _all, highlighting on field2");
         source = searchSource()
@@ -364,7 +364,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
         // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0].string(), equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
 
         logger.info("--> searching on _all, highlighting on field2");
         source = searchSource()
@@ -377,7 +377,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().totalHits(), equalTo(1l));
 
         // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
-        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
+        assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0].string(), equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
     }
 
     @Test
@@ -412,7 +412,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
         for (SearchHit hit : searchResponse.hits()) {
             // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
-            assertThat(hit.highlightFields().get("field1").fragments()[0], equalTo("<em>test</em> " + hit.id()));
+            assertThat(hit.highlightFields().get("field1").fragments()[0].string(), equalTo("<em>test</em> " + hit.id()));
         }
 
         logger.info("--> searching explicitly on field1 and highlighting on it, with DFS");
@@ -425,7 +425,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
         assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
         for (SearchHit hit : searchResponse.hits()) {
-            assertThat(hit.highlightFields().get("field1").fragments()[0], equalTo("<em>test</em> " + hit.id()));
+            assertThat(hit.highlightFields().get("field1").fragments()[0].string(), equalTo("<em>test</em> " + hit.id()));
         }
 
         logger.info("--> searching explicitly _all and highlighting on _all");
@@ -437,7 +437,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
         assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
         for (SearchHit hit : searchResponse.hits()) {
-            assertThat(hit.highlightFields().get("_all").fragments()[0], equalTo("<em>test</em> " + hit.id() + " "));
+            assertThat(hit.highlightFields().get("_all").fragments()[0].string(), equalTo("<em>test</em> " + hit.id() + " "));
         }
     }
 
@@ -481,7 +481,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
 
         for (SearchHit hit : search.hits()) {
             // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
-            assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
         }
     }
 
@@ -516,7 +516,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
 
         for (SearchHit hit : search.hits()) {
             // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
-            assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting <em>bug</em> present in elasticsearch"));
+            assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("highlighting <em>bug</em> present in elasticsearch"));
         }
     }
 
@@ -554,7 +554,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
 
         for (SearchHit hit : search.hits()) {
             // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
-            assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a html escaping highlighting <em>test</em> for *&amp;? elasticsearch"));
+            assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("This is a html escaping highlighting <em>test</em> for *&amp;? elasticsearch"));
         }
     }
 
@@ -591,7 +591,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(search.hits().hits().length, equalTo(5));
 
         for (SearchHit hit : search.hits()) {
-            assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting <em>test</em> for *&amp;? elasticsearch"));
+            assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("highlighting <em>test</em> for *&amp;? elasticsearch"));
         }
     }
 
@@ -622,7 +622,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         SearchHit hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a <em>test</em>"));
+        assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("this is a <em>test</em>"));
 
         // search on title.key and highlight on title
         search = client.prepareSearch()
@@ -633,7 +633,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        assertThat(hit.highlightFields().get("title.key").fragments()[0].string(), equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
     }
 
     @Test
@@ -663,7 +663,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         SearchHit hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a <em>test</em>"));
+        assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("this is a <em>test</em>"));
 
         // search on title.key and highlight on title.key
         search = client.prepareSearch()
@@ -674,7 +674,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        assertThat(hit.highlightFields().get("title.key").fragments()[0].string(), equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
     }
 
     @Test
@@ -704,7 +704,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         SearchHit hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a <em>test</em>"));
+        assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("this is a <em>test</em>"));
 
         // search on title.key and highlight on title
         search = client.prepareSearch()
@@ -715,7 +715,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        assertThat(hit.highlightFields().get("title.key").fragments()[0].string(), equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
     }
 
     @Test
@@ -745,7 +745,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         SearchHit hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a <em>test</em>"));
+        assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("this is a <em>test</em>"));
 
         // search on title.key and highlight on title.key
         search = client.prepareSearch()
@@ -756,6 +756,6 @@ public class HighlighterSearchTests extends AbstractNodesTests {
         assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0));
 
         hit = search.hits().getAt(0);
-        assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        assertThat(hit.highlightFields().get("title.key").fragments()[0].string(), equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
     }
 }