diff --git a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java index cafd5c485da..24478fd2a6e 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamInput.java @@ -113,6 +113,11 @@ public abstract class AdapterStreamInput extends StreamInput { return in.readText(); } + @Override + public Text readSharedText() throws IOException { + return in.readSharedText(); + } + @Override public int read(byte[] b) throws IOException { return in.read(b); diff --git a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java index 5fc112c404c..b6a4fee3a6e 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/AdapterStreamOutput.java @@ -141,6 +141,11 @@ public class AdapterStreamOutput extends StreamOutput { out.writeText(text); } + @Override + public void writeSharedText(Text text) throws IOException { + out.writeSharedText(text); + } + @Override public void writeFloat(float v) throws IOException { out.writeFloat(v); diff --git a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java index 3a7537c70b0..3d6ea23b23d 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamInput.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.io.stream; import gnu.trove.map.hash.TIntObjectHashMap; +import org.elasticsearch.common.text.Text; import java.io.IOException; @@ -29,9 +30,10 @@ import java.io.IOException; public class HandlesStreamInput extends AdapterStreamInput { private final TIntObjectHashMap handles = new TIntObjectHashMap(); - private final TIntObjectHashMap identityHandles = new TIntObjectHashMap(); + private final TIntObjectHashMap handlesText = new TIntObjectHashMap(); + HandlesStreamInput() { super(); } @@ -89,21 +91,41 @@ public class HandlesStreamInput extends AdapterStreamInput { } } + @Override + public Text readSharedText() throws IOException { + byte b = in.readByte(); + if (b == 0) { + int handle = in.readVInt(); + Text s = in.readText(); + handlesText.put(handle, s); + return s; + } else if (b == 1) { + return handlesText.get(in.readVInt()); + } else if (b == 2) { + return in.readText(); + } else { + throw new IOException("Expected handle header, got [" + b + "]"); + } + } + @Override public void reset() throws IOException { super.reset(); handles.clear(); identityHandles.clear(); + handlesText.clear(); } public void reset(StreamInput in) { super.reset(in); handles.clear(); identityHandles.clear(); + handlesText.clear(); } public void cleanHandles() { handles.clear(); identityHandles.clear(); + handlesText.clear(); } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java index b4a364e0b33..136fda0a249 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/HandlesStreamOutput.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.io.stream; import gnu.trove.impl.Constants; import gnu.trove.map.hash.TObjectIntHashMap; +import org.elasticsearch.common.text.Text; import java.io.IOException; import java.util.Arrays; @@ -36,9 +37,10 @@ public class HandlesStreamOutput extends AdapterStreamOutput { private final int identityThreshold; private final TObjectIntHashMap handles = new TObjectIntHashMap(Constants.DEFAULT_CAPACITY, Constants.DEFAULT_LOAD_FACTOR, -1); - private final HandleTable identityHandles = new HandleTable(10, (float) 3.00); + private final TObjectIntHashMap handlesText = new TObjectIntHashMap(Constants.DEFAULT_CAPACITY, Constants.DEFAULT_LOAD_FACTOR, -1); + public HandlesStreamOutput(StreamOutput out) { this(out, DEFAULT_IDENTITY_THRESHOLD); } @@ -105,10 +107,37 @@ public class HandlesStreamOutput extends AdapterStreamOutput { } } + @Override + public void writeSharedText(Text text) throws IOException { + int length; + if (text.hasBytes()) { + length = text.bytes().length(); + } else { + length = text.string().length(); + } + if (length < identityThreshold) { + int handle = handlesText.get(text); + if (handle == -1) { + handle = handlesText.size(); + handlesText.put(text, handle); + out.writeByte((byte) 0); + out.writeVInt(handle); + out.writeText(text); + } else { + out.writeByte((byte) 1); + out.writeVInt(handle); + } + } else { + out.writeByte((byte) 2); + out.writeText(text); + } + } + @Override public void reset() throws IOException { handles.clear(); identityHandles.clear(); + handlesText.clear(); if (out != null) { out.reset(); } @@ -117,6 +146,7 @@ public class HandlesStreamOutput extends AdapterStreamOutput { public void clear() { handles.clear(); identityHandles.clear(); + handlesText.clear(); } /** diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index b993c9a55e0..e3ed22c4bb6 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -198,6 +198,10 @@ public abstract class StreamInput extends InputStream { return new StringAndBytesText(readBytesReference(length)); } + public Text readSharedText() throws IOException { + return readText(); + } + @Nullable public String readOptionalString() throws IOException { if (readBoolean()) { diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 5af27273970..283576120b8 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -204,6 +204,10 @@ public abstract class StreamOutput extends OutputStream { } } + public void writeSharedText(Text text) throws IOException { + writeText(text); + } + public void writeString(String str) throws IOException { int charCount = str.length(); writeVInt(charCount); diff --git a/src/main/java/org/elasticsearch/common/text/BytesText.java b/src/main/java/org/elasticsearch/common/text/BytesText.java index dc8ea88e219..488490b70ba 100644 --- a/src/main/java/org/elasticsearch/common/text/BytesText.java +++ b/src/main/java/org/elasticsearch/common/text/BytesText.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesReference; public class BytesText implements Text { private BytesReference bytes; + private int hash; public BytesText(BytesReference bytes) { this.bytes = bytes; @@ -64,7 +65,10 @@ public class BytesText implements Text { @Override public int hashCode() { - return bytes().hashCode(); + if (hash == 0) { + hash = bytes.hashCode(); + } + return hash; } @Override diff --git a/src/main/java/org/elasticsearch/common/text/StringAndBytesText.java b/src/main/java/org/elasticsearch/common/text/StringAndBytesText.java index 73e2d196e49..ea26e7dc17d 100644 --- a/src/main/java/org/elasticsearch/common/text/StringAndBytesText.java +++ b/src/main/java/org/elasticsearch/common/text/StringAndBytesText.java @@ -44,6 +44,7 @@ public class StringAndBytesText implements Text { private BytesReference bytes; private String text; + private int hash; public StringAndBytesText(BytesReference bytes) { this.bytes = bytes; @@ -90,7 +91,10 @@ public class StringAndBytesText implements Text { @Override public int hashCode() { - return bytes().hashCode(); + if (hash == 0) { + hash = bytes().hashCode(); + } + return hash; } @Override diff --git a/src/main/java/org/elasticsearch/common/text/StringText.java b/src/main/java/org/elasticsearch/common/text/StringText.java index c478396b35d..cda2a9ec9ba 100644 --- a/src/main/java/org/elasticsearch/common/text/StringText.java +++ b/src/main/java/org/elasticsearch/common/text/StringText.java @@ -42,6 +42,7 @@ public class StringText implements Text { } private final String text; + private int hash; public StringText(String text) { this.text = text; @@ -75,7 +76,10 @@ public class StringText implements Text { @Override public int hashCode() { // we use bytes here so we can be consistent with other text implementations - return bytes().hashCode(); + if (hash == 0) { + hash = bytes().hashCode(); + } + return hash; } @Override diff --git a/src/main/java/org/elasticsearch/common/text/Text.java b/src/main/java/org/elasticsearch/common/text/Text.java index 03dc18bfb7c..a009092d7ad 100644 --- a/src/main/java/org/elasticsearch/common/text/Text.java +++ b/src/main/java/org/elasticsearch/common/text/Text.java @@ -21,12 +21,14 @@ package org.elasticsearch.common.text; import org.elasticsearch.common.bytes.BytesReference; +import java.io.Serializable; + /** * Text represents a (usually) long text data. We use this abstraction instead of {@link String} * so we can represent it in a more optimized manner in memory as well as serializing it over the * network as well as converting it to json format. */ -public interface Text extends Comparable { +public interface Text extends Comparable, Serializable { /** * Are bytes available without the need to be converted into bytes when calling {@link #bytes()}. diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 31732e32e31..4d12f2cc35a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -35,6 +35,8 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.text.StringAndBytesText; +import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.*; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.internal.*; @@ -237,6 +239,7 @@ public class DocumentMapper implements ToXContent { private final Settings indexSettings; private final String type; + private final StringAndBytesText typeText; private final DocumentMapperParser docMapperParser; @@ -279,6 +282,7 @@ public class DocumentMapper implements ToXContent { this.index = index; this.indexSettings = indexSettings; this.type = rootObjectMapper.name(); + this.typeText = new StringAndBytesText(this.type); this.docMapperParser = docMapperParser; this.meta = meta; this.rootObjectMapper = rootObjectMapper; @@ -341,6 +345,10 @@ public class DocumentMapper implements ToXContent { return this.type; } + public Text typeText() { + return this.typeText; + } + public ImmutableMap meta() { return this.meta; } diff --git a/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/src/main/java/org/elasticsearch/search/SearchShardTarget.java index e7ba7e07055..e0fe1bc8773 100644 --- a/src/main/java/org/elasticsearch/search/SearchShardTarget.java +++ b/src/main/java/org/elasticsearch/search/SearchShardTarget.java @@ -23,21 +23,19 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.text.StringAndBytesText; +import org.elasticsearch.common.text.Text; import java.io.IOException; import java.io.Serializable; /** * The target that the search request was executed on. - * - * */ public class SearchShardTarget implements Streamable, Serializable, Comparable { - private String nodeId; - - private String index; - + private Text nodeId; + private Text index; private int shardId; private SearchShardTarget() { @@ -45,27 +43,35 @@ public class SearchShardTarget implements Streamable, Serializable, Comparable fields) { + public InternalSearchHit(int docId, String id, Text type, BytesReference source, Map fields) { this.docId = docId; - this.id = id; + this.id = new StringAndBytesText(id); this.type = type; this.source = source; this.fields = fields; @@ -147,7 +146,7 @@ public class InternalSearchHit implements SearchHit { @Override public String id() { - return id; + return id.string(); } @Override @@ -157,7 +156,7 @@ public class InternalSearchHit implements SearchHit { @Override public String type() { - return type; + return type.string(); } @Override @@ -381,9 +380,9 @@ public class InternalSearchHit implements SearchHit { builder.startObject(); if (explanation() != null) { builder.field("_shard", shard.shardId()); - builder.field("_node", shard.nodeId()); + builder.field("_node", shard.nodeIdText()); } - builder.field(Fields._INDEX, shard.index()); + builder.field(Fields._INDEX, shard.indexText()); builder.field(Fields._TYPE, type); builder.field(Fields._ID, id); if (version != -1) { @@ -482,8 +481,8 @@ public class InternalSearchHit implements SearchHit { public void readFrom(StreamInput in, InternalSearchHits.StreamContext context) throws IOException { score = in.readFloat(); - id = in.readString(); - type = in.readString(); + id = in.readText(); + type = in.readSharedText(); version = in.readLong(); source = in.readBytesReference(); if (source.length() == 0) { @@ -617,8 +616,8 @@ public class InternalSearchHit implements SearchHit { public void writeTo(StreamOutput out, InternalSearchHits.StreamContext context) throws IOException { out.writeFloat(score); - out.writeString(id); - out.writeString(type); + out.writeText(id); + out.writeSharedText(type); out.writeLong(version); out.writeBytesReference(source); if (explanation == null) {