From 46ff97af5e05034de8f171945f0847a98c15c5e3 Mon Sep 17 00:00:00 2001 From: kimchy Date: Tue, 23 Feb 2010 21:30:10 +0200 Subject: [PATCH] refactor json handling to use byte[] instead of string for better performance, storage, and memory consumption (apply to search) --- .../action/search/SearchRequest.java | 16 +++++++++------- .../rest/action/search/RestSearchAction.java | 4 ++-- .../org/elasticsearch/search/SearchService.java | 3 +-- .../search/builder/SearchSourceBuilder.java | 8 ++++---- .../search/internal/InternalSearchRequest.java | 14 ++++++++------ .../search/internal/SearchContext.java | 6 +++--- .../search/TransportTwoServersSearchTests.java | 3 ++- 7 files changed, 29 insertions(+), 25 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/search/SearchRequest.java index d2e5a970076..0f6f59ee91d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -50,7 +50,7 @@ public class SearchRequest implements ActionRequest { private String queryHint; - private String source; + private byte[] source; private Scroll scroll; @@ -78,7 +78,7 @@ public class SearchRequest implements ActionRequest { this(index, source.build()); } - public SearchRequest(String index, String source) { + public SearchRequest(String index, byte[] source) { this(new String[]{index}, source); } @@ -86,7 +86,7 @@ public class SearchRequest implements ActionRequest { this(indices, source.build()); } - public SearchRequest(String[] indices, String source) { + public SearchRequest(String[] indices, byte[] source) { this.indices = indices; this.source = source; } @@ -126,7 +126,7 @@ public class SearchRequest implements ActionRequest { return source(sourceBuilder.build()); } - @Required public SearchRequest source(String source) { + @Required public SearchRequest source(byte[] source) { this.source = source; return this; } @@ -148,7 +148,7 @@ public class SearchRequest implements ActionRequest { return queryHint; } - public String source() { + public byte[] source() { return source; } @@ -234,7 +234,8 @@ public class SearchRequest implements ActionRequest { if (in.readBoolean()) { timeout = readTimeValue(in); } - source = in.readUTF(); + source = new byte[in.readInt()]; + in.readFully(source); int size = in.readInt(); if (size == 0) { @@ -285,7 +286,8 @@ public class SearchRequest implements ActionRequest { out.writeBoolean(true); timeout.writeTo(out); } - out.writeUTF(source); + out.writeInt(source.length); + out.write(source); if (indexBoost == null) { out.writeInt(0); } else { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 0e25cdeca18..3bad3295fea 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -181,9 +181,9 @@ public class RestSearchAction extends BaseRestHandler { return searchRequest; } - private String parseSearchSource(RestRequest request) { + private byte[] parseSearchSource(RestRequest request) { if (request.hasContent()) { - return request.contentAsString(); + return request.contentAsBytes(); } String queryString = request.param("q"); if (queryString == null) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java index 28456ccbf7a..1aa453486bf 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java @@ -51,7 +51,6 @@ import org.elasticsearch.util.component.AbstractComponent; import org.elasticsearch.util.component.Lifecycle; import org.elasticsearch.util.component.LifecycleComponent; import org.elasticsearch.util.concurrent.highscalelib.NonBlockingHashMapLong; -import org.elasticsearch.util.io.FastStringReader; import org.elasticsearch.util.json.Jackson; import org.elasticsearch.util.settings.Settings; import org.elasticsearch.util.timer.Timeout; @@ -290,7 +289,7 @@ public class SearchService extends AbstractComponent implements LifecycleCompone private void parseSource(SearchContext context) throws SearchParseException { try { - JsonParser jp = jsonFactory.createJsonParser(new FastStringReader(context.source())); + JsonParser jp = jsonFactory.createJsonParser(context.source()); JsonToken token; while ((token = jp.nextToken()) != JsonToken.END_OBJECT) { if (token == JsonToken.FIELD_NAME) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index db4b40a7822..3902ca4f8d2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -21,7 +21,7 @@ package org.elasticsearch.search.builder; import org.elasticsearch.index.query.json.JsonQueryBuilder; import org.elasticsearch.search.SearchException; -import org.elasticsearch.util.json.StringJsonBuilder; +import org.elasticsearch.util.json.JsonBuilder; import org.elasticsearch.util.json.ToJson; import java.util.ArrayList; @@ -125,9 +125,9 @@ public class SearchSourceBuilder { return this; } - public String build() throws SearchException { + public byte[] build() throws SearchException { try { - StringJsonBuilder builder = stringJsonBuilder(); + JsonBuilder builder = binaryJsonBuilder(); builder.startObject(); if (from != -1) { @@ -182,7 +182,7 @@ public class SearchSourceBuilder { builder.endObject(); - return builder.string(); + return builder.copiedBytes(); } catch (Exception e) { throw new SearchSourceBuilderException("Failed to build search source", e); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java index b2f076996fd..a2fb5de2b9e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/InternalSearchRequest.java @@ -70,16 +70,16 @@ public class InternalSearchRequest implements Streamable { private String[] types = Strings.EMPTY_ARRAY; - private String source; + private byte[] source; public InternalSearchRequest() { } - public InternalSearchRequest(ShardRouting shardRouting, String source) { + public InternalSearchRequest(ShardRouting shardRouting, byte[] source) { this(shardRouting.index(), shardRouting.id(), source); } - public InternalSearchRequest(String index, int shardId, String source) { + public InternalSearchRequest(String index, int shardId, byte[] source) { this.index = index; this.shardId = shardId; this.source = source; @@ -93,7 +93,7 @@ public class InternalSearchRequest implements Streamable { return shardId; } - public String source() { + public byte[] source() { return this.source; } @@ -165,7 +165,8 @@ public class InternalSearchRequest implements Streamable { if (in.readBoolean()) { timeout = readTimeValue(in); } - source = in.readUTF(); + source = new byte[in.readInt()]; + in.readFully(source); queryBoost = in.readFloat(); int typesSize = in.readInt(); if (typesSize > 0) { @@ -193,7 +194,8 @@ public class InternalSearchRequest implements Streamable { out.writeBoolean(true); timeout.writeTo(out); } - out.writeUTF(source); + out.writeInt(source.length); + out.write(source); out.writeFloat(queryBoost); out.writeInt(types.length); for (String type : types) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 34fe502409b..4f15dcc9667 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -50,7 +50,7 @@ public class SearchContext implements Releasable { private final SearchShardTarget shardTarget; - private final String source; + private final byte[] source; private final Engine.Searcher engineSearcher; @@ -100,7 +100,7 @@ public class SearchContext implements Releasable { private volatile Timeout keepAliveTimeout; - public SearchContext(long id, SearchShardTarget shardTarget, TimeValue timeout, float queryBoost, String source, + public SearchContext(long id, SearchShardTarget shardTarget, TimeValue timeout, float queryBoost, byte[] source, String[] types, Engine.Searcher engineSearcher, IndexService indexService) { this.id = id; this.shardTarget = shardTarget; @@ -138,7 +138,7 @@ public class SearchContext implements Releasable { return this.shardTarget; } - public String source() { + public byte[] source() { return source; } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/TransportTwoServersSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/TransportTwoServersSearchTests.java index 499e540821b..2546c4a7c4e 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/TransportTwoServersSearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/TransportTwoServersSearchTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.integration.AbstractServersTests; +import org.elasticsearch.util.Unicode; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -237,7 +238,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests { @Test public void testFailedSearch() throws Exception { logger.info("Start Testing failed search"); - SearchResponse searchResponse = client.search(searchRequest("test").source("{ xxx }")).actionGet(); + SearchResponse searchResponse = client.search(searchRequest("test").source(Unicode.fromStringAsBytes("{ xxx }"))).actionGet(); assertThat(searchResponse.successfulShards(), equalTo(0)); logger.info("Failures:"); for (ShardSearchFailure searchFailure : searchResponse.shardFailures()) {