honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Store Compression: Term Vector Vector, closes #2049.

This commit is contained in:
Shay Banon 2012-06-23 23:11:00 +02:00
parent ded5b773da
commit 2fb867b467
5 changed files with 44 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java
View File

@ -206,9 +206,10 @@ public abstract class CompressedIndexInput extends IndexInput {
@Override
public Object clone() {
// we clone and we need to make sure we keep the same positions!
CompressedIndexInput cloned = (CompressedIndexInput) super.clone();
cloned.position = 0;
cloned.valid = 0;
cloned.uncompressed = new byte[uncompressed.length];
System.arraycopy(uncompressed, 0, cloned.uncompressed, 0, uncompressed.length);
cloned.in = (IndexInput) cloned.in.clone();
return cloned;
}

2
src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java
View File

@ -66,8 +66,6 @@ public class LZFCompressedIndexInput extends CompressedIndexInput {
@Override
public Object clone() {
LZFCompressedIndexInput cloned = (LZFCompressedIndexInput) super.clone();
cloned.uncompressed = new byte[LZFChunk.MAX_CHUNK_LEN];
System.arraycopy(uncompressed, 0, cloned.uncompressed, 0, uncompressed.length);
cloned.inputBuffer = new byte[LZFChunk.MAX_CHUNK_LEN];
return cloned;
}

30
src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -57,17 +57,23 @@ public class Store extends AbstractIndexShardComponent {
static {
IndexMetaData.addDynamicSettings(
"index.store.compress.stored"
"index.store.compress.stored",
"index.store.compress.tv"
);
}
class ApplySettings implements IndexSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
boolean compressedStoredFields = settings.getAsBoolean("index.store.compress.stored", Store.this.compressedStoredFields);
if (compressedStoredFields != Store.this.compressedStoredFields) {
logger.info("updating [index.store.compress.stored] from [{}] to [{}]", Store.this.compressedStoredFields, compressedStoredFields);
Store.this.compressedStoredFields = compressedStoredFields;
boolean compressStored = settings.getAsBoolean("index.store.compress.stored", Store.this.compressStored);
if (compressStored != Store.this.compressStored) {
logger.info("updating [index.store.compress.stored] from [{}] to [{}]", Store.this.compressStored, compressStored);
Store.this.compressStored = compressStored;
}
boolean compressTv = settings.getAsBoolean("index.store.compress.tv", Store.this.compressTv);
if (compressTv != Store.this.compressTv) {
logger.info("updating [index.store.compress.tv] from [{}] to [{}]", Store.this.compressTv, compressTv);
Store.this.compressTv = compressTv;
}
}
}
@ -95,7 +101,8 @@ public class Store extends AbstractIndexShardComponent {
private final boolean sync;
private volatile boolean compressedStoredFields;
private volatile boolean compressStored;
private volatile boolean compressTv;
private final ApplySettings applySettings = new ApplySettings();
@ -109,9 +116,10 @@ public class Store extends AbstractIndexShardComponent {
this.sync = componentSettings.getAsBoolean("sync", true); // TODO we don't really need to fsync when using shared gateway...
this.directory = new StoreDirectory(directoryService.build());
this.compressedStoredFields = componentSettings.getAsBoolean("compress.stored", false);
this.compressStored = componentSettings.getAsBoolean("compress.stored", false);
this.compressTv = componentSettings.getAsBoolean("compress.tv", false);
logger.debug("using compress.stored [{}]", compressedStoredFields);
logger.debug("using compress.stored [{}], compress.tv [{}]", compressStored, compressTv);
indexSettingsService.addListener(applySettings);
}
@ -480,7 +488,7 @@ public class Store extends AbstractIndexShardComponent {
computeChecksum = false;
}
}
if (!raw && compressedStoredFields && name.endsWith(".fdt")) {
if (!raw && ((compressStored && name.endsWith(".fdt")) || (compressTv && name.endsWith(".tvf")))) {
if (computeChecksum) {
// with compression, there is no need for buffering when doing checksums
// since we have buffering on the compressed index output
@ -503,7 +511,7 @@ public class Store extends AbstractIndexShardComponent {
throw new FileNotFoundException(name);
}
IndexInput in = metaData.directory().openInput(name);
if (name.endsWith(".fdt")) {
if (name.endsWith(".fdt") || name.endsWith(".tvf")) {
Compressor compressor = CompressorFactory.compressor(in);
if (compressor != null) {
in = compressor.indexInput(in);
@ -519,7 +527,7 @@ public class Store extends AbstractIndexShardComponent {
throw new FileNotFoundException(name);
}
IndexInput in = metaData.directory().openInput(name, bufferSize);
if (name.endsWith(".fdt")) {
if (name.endsWith(".fdt") || name.endsWith(".tvf")) {
Compressor compressor = CompressorFactory.compressor(in);
if (compressor != null) {
in = compressor.indexInput(in);

19
src/test/java/org/elasticsearch/benchmark/compress/LuceneCompressionBenchmark.java
View File

@ -45,6 +45,7 @@ public class LuceneCompressionBenchmark {
public static void main(String[] args) throws Exception {
final long MAX_SIZE = ByteSizeValue.parseBytesSizeValue("50mb").bytes();
final boolean WITH_TV = true;
final Compressor compressor = CompressorFactory.defaultCompressor();
@ -61,6 +62,9 @@ public class LuceneCompressionBenchmark {
if (name.endsWith(".fdt")) {
return compressor.indexOutput(super.createOutput(name));
}
if (WITH_TV && name.endsWith(".tvf")) {
return compressor.indexOutput(super.createOutput(name));
}
return super.createOutput(name);
}
@ -75,12 +79,21 @@ public class LuceneCompressionBenchmark {
return in;
}
}
if (WITH_TV && name.endsWith(".tvf")) {
IndexInput in = super.openInput(name);
Compressor compressor1 = CompressorFactory.compressor(in);
if (compressor1 != null) {
return compressor1.indexInput(in);
} else {
return in;
}
}
return super.openInput(name);
}
@Override
public IndexInput openInput(String name, int bufferSize) throws IOException {
if (name.endsWith(".fdt")) {
if (name.endsWith(".fdt") || name.endsWith(".tvf")) {
IndexInput in = super.openInput(name, bufferSize);
// in case the override called openInput(String)
if (in instanceof CompressedIndexInput) {
@ -108,6 +121,10 @@ public class LuceneCompressionBenchmark {
builder.close();
Document doc = new Document();
doc.add(new Field("_source", builder.underlyingBytes(), 0, builder.underlyingBytesLength()));
if (WITH_TV) {
Field field = new Field("text", builder.string(), Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
doc.add(field);
}
uncompressedWriter.addDocument(doc);
compressedWriter.addDocument(doc);
}

4
src/test/java/org/elasticsearch/benchmark/compress/TestData.java
View File

@ -69,6 +69,10 @@ public class TestData {
return true;
}
public String currentText() {
return text;
}
/**
*/
public XContentBuilder current(XContentBuilder builder) throws Exception {