honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Search: Reduce memory usage during fetch source sub phase. 

If includes or excludes are set
XContentFactory.xcontentBuilder() allocates a new
BytesStreamOutput using the default page size which is 16kb.

Can be optimized to use the length of the sourceRef because
that is the maximum possible size that the streamOutput will
use.

This redcues the amount of memory allocated for a request
that is fetching 200.000 small documents (~150 bytes each)
by about 300 MB

Close #8138
This commit is contained in:
Mathias Fussenegger 2014-10-17 15:11:53 +02:00 committed by Adrien Grand
parent 5eeac2fdf6
commit b4cad96597
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java
View File

@ -22,12 +22,14 @@ package org.elasticsearch.search.fetch.source;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.InternalSearchHit;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.lookup.SourceLookup;
import java.io.IOException;
import java.util.Map;
@ -71,9 +73,12 @@ public class FetchSourceSubPhase implements FetchSubPhase {
return;
}
Object value = context.lookup().source().filter(fetchSourceContext.includes(), fetchSourceContext.excludes());
SourceLookup source = context.lookup().source();
Object value = source.filter(fetchSourceContext.includes(), fetchSourceContext.excludes());
try {
XContentBuilder builder = XContentFactory.contentBuilder(context.lookup().source().sourceContentType());
final int initialCapacity = Math.min(1024, source.internalSourceRef().length());
BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity);
XContentBuilder builder = new XContentBuilder(context.lookup().source().sourceContentType().xContent(), streamOutput);
builder.value(value);
hitContext.hit().sourceRef(builder.bytes());
} catch (IOException e) {