honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apping: add threshold to _source field compression, closes #506.

This commit is contained in:
kimchy 2010-11-10 21:03:54 +02:00
parent a2e674df49
commit 70a0e110d4
5 changed files with 129 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/elasticsearch/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
View File

@ -123,6 +123,10 @@ public class ByteSizeValue implements Serializable, Streamable {
return Strings.format1Decimals(value, suffix);
}
public static ByteSizeValue parseBytesSizeValue(String sValue) throws ElasticSearchParseException {
return parseBytesSizeValue(sValue, null);
}
public static ByteSizeValue parseBytesSizeValue(String sValue, ByteSizeValue defaultValue) throws ElasticSearchParseException {
if (sValue == null) {
return defaultValue;

28
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/SourceFieldMapper.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.common.compress.lzf.LZFDecoder;
import org.elasticsearch.common.compress.lzf.LZFEncoder;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.MergeMappingException;
@ -39,6 +40,7 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
public static class Defaults extends AbstractFieldMapper.Defaults {
public static final String NAME = org.elasticsearch.index.mapper.SourceFieldMapper.NAME;
public static final boolean ENABLED = true;
public static final long COMPRESS_THRESHOLD = -1;
public static final Field.Index INDEX = Field.Index.NO;
public static final Field.Store STORE = Field.Store.YES;
public static final boolean OMIT_NORMS = true;
@ -49,6 +51,8 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
private boolean enabled = Defaults.ENABLED;
private long compressThreshold = -1;
private Boolean compress = null;
public Builder() {
@ -65,8 +69,13 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
return this;
}
public Builder compressThreshold(long compressThreshold) {
this.compressThreshold = compressThreshold;
return this;
}
@Override public SourceFieldMapper build(BuilderContext context) {
return new SourceFieldMapper(name, enabled, compress);
return new SourceFieldMapper(name, enabled, compress, compressThreshold);
}
}
@ -74,17 +83,20 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
private Boolean compress;
private long compressThreshold;
private final SourceFieldSelector fieldSelector;
protected SourceFieldMapper() {
this(Defaults.NAME, Defaults.ENABLED, null);
this(Defaults.NAME, Defaults.ENABLED, null, -1);
}
protected SourceFieldMapper(String name, boolean enabled, Boolean compress) {
protected SourceFieldMapper(String name, boolean enabled, Boolean compress, long compressThreshold) {
super(new Names(name, name, name, name), Defaults.INDEX, Defaults.STORE, Defaults.TERM_VECTOR, Defaults.BOOST,
Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER);
this.enabled = enabled;
this.compress = compress;
this.compressThreshold = compressThreshold;
this.fieldSelector = new SourceFieldSelector(names.indexName());
}
@ -106,8 +118,10 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
}
byte[] data = context.source();
if (compress != null && compress) {
if (compressThreshold == -1 || data.length > compressThreshold) {
data = LZFEncoder.encodeWithCache(data, data.length);
}
}
return new Field(names.indexName(), data, store);
}
@ -169,7 +183,7 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
@Override public void toXContent(XContentBuilder builder, Params params) throws IOException {
// all are defaults, no need to write it at all
if (enabled == Defaults.ENABLED && compress == null) {
if (enabled == Defaults.ENABLED && compress == null && compressThreshold == -1) {
return;
}
builder.startObject(contentType());
@ -179,6 +193,9 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
if (compress != null) {
builder.field("compress", compress);
}
if (compressThreshold != -1) {
builder.field("compress_threshold", new ByteSizeValue(compressThreshold).toString());
}
builder.endObject();
}
@ -188,6 +205,9 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements or
if (sourceMergeWith.compress != null) {
this.compress = sourceMergeWith.compress;
}
if (sourceMergeWith.compressThreshold != -1) {
this.compressThreshold = sourceMergeWith.compressThreshold;
}
}
}
}

9
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
@ -246,6 +247,14 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("compress") && fieldNode != null) {
builder.compress(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("compress_threshold") && fieldNode != null) {
if (fieldNode instanceof Number) {
builder.compressThreshold(((Number) fieldNode).longValue());
builder.compress(true);
} else {
builder.compressThreshold(ByteSizeValue.parseBytesSizeValue(fieldNode.toString()).bytes());
builder.compress(true);
}
}
}
return builder;

90
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/source/CompressSourceMappingTests.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.xcontent.source;
import org.elasticsearch.common.compress.lzf.LZFDecoder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.xcontent.MapperTests;
import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
import org.testng.annotations.Test;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/**
* @author kimchy (shay.banon)
*/
public class CompressSourceMappingTests {
@Test public void testCompressDisabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("compress", false).endObject()
.endObject().endObject().string();
XContentDocumentMapper documentMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
.field("field2", "value2")
.endObject().copiedBytes());
assertThat(LZFDecoder.isCompressed(doc.doc().getBinaryValue("_source")), equalTo(false));
}
@Test public void testCompressEnabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("compress", true).endObject()
.endObject().endObject().string();
XContentDocumentMapper documentMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
.field("field2", "value2")
.endObject().copiedBytes());
assertThat(LZFDecoder.isCompressed(doc.doc().getBinaryValue("_source")), equalTo(true));
}
@Test public void testCompressThreshold() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("compress_threshold", "200b").endObject()
.endObject().endObject().string();
XContentDocumentMapper documentMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
.endObject().copiedBytes());
assertThat(LZFDecoder.isCompressed(doc.doc().getBinaryValue("_source")), equalTo(false));
doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
.field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz")
.field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz")
.field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz")
.field("field2", "value2 xxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzz")
.endObject().copiedBytes());
assertThat(LZFDecoder.isCompressed(doc.doc().getBinaryValue("_source")), equalTo(true));
}
}

2
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/defaultsource/DefaultSourceMappingTests.java → modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/source/DefaultSourceMappingTests.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.elasticsearch.index.mapper.xcontent.defaultsource;
package org.elasticsearch.index.mapper.xcontent.source;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;