honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BytesStreamOutput default size should be 2k instead of 32k 

We changed the default of BytesStreamOutput (used in various places in ES) to 32k from 1k with the assumption that most stream tend to be large. This doesn't hold for example when indexing small documents and adding them using XContentBuilder (which will have a large overhead).

Default the buffer size to 2k now, but be relatively aggressive in expanding the buffer when below 256k (double it), and just use oversize (1/8th) when larger to try and minimize garbage and buffer copies.

relates to #3624
closes #3638
This commit is contained in:
Shay Banon 2013-09-06 02:25:17 +02:00
parent 3e92b1d6b8
commit 4155741f7f
2 changed files with 31 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/main/java/org/elasticsearch/common/io/stream/BytesStreamOutput.java
View File

@ -31,7 +31,9 @@ import java.io.IOException;
*/
public class BytesStreamOutput extends StreamOutput implements BytesStream {
public static final int DEFAULT_SIZE = 32 * 1024;
public static final int DEFAULT_SIZE = 2 * 1024;
public static final int OVERSIZE_LIMIT = 256 * 1024;
/**
* The buffer where data is stored.
@ -73,7 +75,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
public void writeByte(byte b) throws IOException {
int newcount = count + 1;
if (newcount > buf.length) {
buf = ArrayUtil.grow(buf, newcount);
buf = grow(newcount);
}
buf[count] = b;
count = newcount;
@ -82,7 +84,7 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
public void skip(int length) {
int newcount = count + length;
if (newcount > buf.length) {
buf = ArrayUtil.grow(buf, newcount);
buf = grow(newcount);
}
count = newcount;
}
@ -94,12 +96,20 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
}
int newcount = count + length;
if (newcount > buf.length) {
buf = ArrayUtil.grow(buf, newcount);
buf = grow(newcount);
}
System.arraycopy(b, offset, buf, count, length);
count = newcount;
}
private byte[] grow(int newCount) {
// try and grow faster while we are small...
if (newCount < OVERSIZE_LIMIT) {
newCount = Math.max(buf.length << 1, newCount);
}
return ArrayUtil.grow(buf, newCount);
}
public void seek(int seekTo) {
count = seekTo;
}
@ -108,6 +118,10 @@ public class BytesStreamOutput extends StreamOutput implements BytesStream {
count = 0;
}
public int bufferSize() {
return buf.length;
}
@Override
public void flush() throws IOException {
// nothing to do there

13
src/test/java/org/elasticsearch/test/unit/common/io/streams/BytesStreamsTests.java
View File

@ -60,4 +60,17 @@ public class BytesStreamsTests {
assertThat(in.readString(), equalTo("hello"));
assertThat(in.readString(), equalTo("goodbye"));
}
@Test
public void testGrowLogic() throws Exception {
BytesStreamOutput out = new BytesStreamOutput();
out.writeBytes(new byte[BytesStreamOutput.DEFAULT_SIZE - 5]);
assertThat(out.bufferSize(), equalTo(2048)); // remains the default
out.writeBytes(new byte[1 * 1024]);
assertThat(out.bufferSize(), equalTo(4608));
out.writeBytes(new byte[32 * 1024]);
assertThat(out.bufferSize(), equalTo(40320));
out.writeBytes(new byte[32 * 1024]);
assertThat(out.bufferSize(), equalTo(90720));
}
}