honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-09 14:34:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

test some linedocs data with compressed streams

Browse Source
This commit is contained in:
Robert Muir 2014-08-07 17:20:41 -04:00
parent 975327775a
commit d341274457
1 changed files with 54 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/test/java/org/elasticsearch/common/compress/CompressedStreamTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common.compress;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.TestUtil;
import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
@ -30,6 +31,7 @@ import org.elasticsearch.test.ElasticsearchTestCase;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Random;
/**
@ -38,42 +40,61 @@ import java.util.Random;
public class CompressedStreamTests extends ElasticsearchTestCase {
public void testRandom() throws IOException {
String compressor = "lzf";
CompressorFactory.configure(ImmutableSettings.settingsBuilder().put("compress.default.type", compressor).build());
Random r = getRandom();
for (int i = 0; i < 2000; i++) {
Random r = getRandom();
byte bytes[] = new byte[TestUtil.nextInt(r, 1, 10000)];
r.nextBytes(bytes);
ByteBuffer bb = ByteBuffer.wrap(bytes);
StreamInput rawIn = new ByteBufferStreamInput(bb);
Compressor c = CompressorFactory.defaultCompressor();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
OutputStreamStreamOutput rawOs = new OutputStreamStreamOutput(bos);
StreamOutput os = c.streamOutput(rawOs);
byte buffer[] = new byte[1024]; //arbitrary
int len;
while ((len = rawIn.read(buffer)) != -1) {
os.write(buffer, 0, len);
}
os.close();
// now we have compressed byte array
byte compressed[] = bos.toByteArray();
ByteBuffer bb2 = ByteBuffer.wrap(compressed);
StreamInput compressedIn = new ByteBufferStreamInput(bb2);
StreamInput in = c.streamInput(compressedIn);
ByteArrayOutputStream uncompressedOut = new ByteArrayOutputStream();
while ((len = in.read(buffer)) != -1) {
uncompressedOut.write(buffer, 0, len);
}
uncompressedOut.close();
assertArrayEquals(bytes, uncompressedOut.toByteArray());
doTest("lzf", bytes);
}
}
public void testLineDocs() throws IOException {
Random r = getRandom();
LineFileDocs lineFileDocs = new LineFileDocs(r);
for (int i = 0; i < 200; i++) {
int numDocs = TestUtil.nextInt(r, 1, 100);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (int j = 0; j < numDocs; j++) {
String s = lineFileDocs.nextDoc().get("body");
bos.write(s.getBytes(StandardCharsets.UTF_8));
}
doTest("lzf", bos.toByteArray());
}
lineFileDocs.close();
}
private void doTest(String compressor, byte bytes[]) throws IOException {
CompressorFactory.configure(ImmutableSettings.settingsBuilder().put("compress.default.type", compressor).build());
ByteBuffer bb = ByteBuffer.wrap(bytes);
StreamInput rawIn = new ByteBufferStreamInput(bb);
Compressor c = CompressorFactory.defaultCompressor();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
OutputStreamStreamOutput rawOs = new OutputStreamStreamOutput(bos);
StreamOutput os = c.streamOutput(rawOs);
int bufferSize = TestUtil.nextInt(getRandom(), 1, 2048);
byte buffer[] = new byte[bufferSize];
int len;
while ((len = rawIn.read(buffer)) != -1) {
os.write(buffer, 0, len);
}
os.close();
rawIn.close();
// now we have compressed byte array
byte compressed[] = bos.toByteArray();
ByteBuffer bb2 = ByteBuffer.wrap(compressed);
StreamInput compressedIn = new ByteBufferStreamInput(bb2);
StreamInput in = c.streamInput(compressedIn);
ByteArrayOutputStream uncompressedOut = new ByteArrayOutputStream();
while ((len = in.read(buffer)) != -1) {
uncompressedOut.write(buffer, 0, len);
}
uncompressedOut.close();
assertArrayEquals(bytes, uncompressedOut.toByteArray());
}
}