honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Binary Mapped Fields: Allow to not store them by default, and return BytesReference 

fixes #2523
also, fix another point of normalization of the result for get API
This commit is contained in:
Shay Banon 2013-01-05 01:50:46 +01:00
parent 4b9fcdb900
commit 0e5287f1f2
4 changed files with 32 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/org/elasticsearch/index/get/ShardGetService.java
View File

@ -244,6 +244,10 @@ public class ShardGetService extends AbstractIndexShardComponent {
// only if the field is stored or source is enabled we should add it..
if (docMapper.sourceMapper().enabled() || x == null || x.fieldType().stored()) {
value = searchLookup.source().extractValue(field);
// normalize the data if needed (mainly for binary fields, to convert from base64 strings to bytes)
if (value != null && x != null) {
value = x.valueForSearch(value);
}
}
}
}
@ -326,7 +330,7 @@ public class ShardGetService extends AbstractIndexShardComponent {
}
value = searchLookup.source().extractValue(field);
// normalize the data if needed (mainly for binary fields, to convert from base64 strings to bytes)
if (value != null) {
if (value != null && x != null) {
value = x.mapper().valueForSearch(value);
}
}

12
src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
View File

@ -46,7 +46,7 @@ import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
/**
*
*/
public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
public class BinaryFieldMapper extends AbstractFieldMapper<BytesReference> {
public static final String CONTENT_TYPE = "binary";
@ -55,7 +55,8 @@ public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
public static final FieldType FIELD_TYPE = new FieldType(AbstractFieldMapper.Defaults.FIELD_TYPE);
static {
FIELD_TYPE.setStored(false);
FIELD_TYPE.setIndexed(false);
FIELD_TYPE.setStored(true);
FIELD_TYPE.freeze();
}
}
@ -137,7 +138,7 @@ public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
}
@Override
public byte[] value(Object value) {
public BytesReference value(Object value) {
if (value == null) {
return null;
}
@ -157,7 +158,7 @@ public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
}
}
try {
return CompressorFactory.uncompressIfNeeded(bytes).toBytes();
return CompressorFactory.uncompressIfNeeded(bytes);
} catch (IOException e) {
throw new ElasticSearchParseException("failed to decompress source", e);
}
@ -210,6 +211,9 @@ public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
if (compressThreshold != -1) {
builder.field("compress_threshold", new ByteSizeValue(compressThreshold).toString());
}
if (fieldType.stored() != defaultFieldType().stored()) {
builder.field("store", fieldType.stored());
}
builder.endObject();
return builder;
}

25
src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java
View File

@ -26,7 +26,10 @@ import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.integration.AbstractNodesTests;
@ -39,8 +42,7 @@ import java.util.List;
import static org.elasticsearch.client.Requests.clusterHealthRequest;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.*;
public class GetActionTests extends AbstractNodesTests {
@ -238,6 +240,7 @@ public class GetActionTests extends AbstractNodesTests {
.startObject("str").field("type", "string").field("store", "yes").endObject()
.startObject("int").field("type", "integer").field("store", "yes").endObject()
.startObject("date").field("type", "date").field("store", "yes").endObject()
.startObject("binary").field("type", "binary").field("store", "yes").endObject()
.endObject()
.endObject().endObject())
.execute().actionGet();
@ -246,40 +249,44 @@ public class GetActionTests extends AbstractNodesTests {
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));
client.prepareIndex("test", "type1", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z").execute().actionGet();
client.prepareIndex("test", "type2", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z").execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z", "binary", Base64.encodeBytes(new byte[]{1, 2, 3})).execute().actionGet();
client.prepareIndex("test", "type2", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z", "binary", Base64.encodeBytes(new byte[]{1, 2, 3})).execute().actionGet();
// realtime get with stored source
logger.info("--> realtime get (from source)");
GetResponse getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date").execute().actionGet();
GetResponse getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Integer) getResponse.field("int").getValue(), equalTo(42));
assertThat((Long) getResponse.field("int").getValue(), equalTo(42l));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat(getResponse.field("binary").getValue(), instanceOf(String.class)); // its a String..., not binary mapped
logger.info("--> realtime get (from stored fields)");
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date").execute().actionGet();
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Integer) getResponse.field("int").getValue(), equalTo(42));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat((BytesReference) getResponse.field("binary").getValue(), equalTo((BytesReference) new BytesArray(new byte[]{1, 2, 3})));
logger.info("--> flush the index, so we load it from it");
client.admin().indices().prepareFlush().execute().actionGet();
logger.info("--> non realtime get (from source)");
getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date").execute().actionGet();
getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Long) getResponse.field("int").getValue(), equalTo(42l));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat(getResponse.field("binary").getValue(), instanceOf(String.class)); // its a String..., not binary mapped
logger.info("--> non realtime get (from stored fields)");
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date").execute().actionGet();
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Integer) getResponse.field("int").getValue(), equalTo(42));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat((BytesReference) getResponse.field("binary").getValue(), equalTo((BytesReference) new BytesArray(new byte[]{1, 2, 3})));
}
@Test

4
src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java
View File

@ -22,6 +22,8 @@ package org.elasticsearch.test.integration.search.fields;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.xcontent.XContentFactory;
@ -343,7 +345,7 @@ public class SearchFieldsTests extends AbstractNodesTests {
String dateTime = Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC));
assertThat(searchResponse.hits().getAt(0).fields().get("date_field").value(), equalTo((Object) dateTime));
assertThat(searchResponse.hits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) Boolean.TRUE));
assertThat(searchResponse.hits().getAt(0).fields().get("binary_field").value().toString(), equalTo(Base64.encodeBytes("testing text".getBytes("UTF8"))));
assertThat(((BytesReference) searchResponse.hits().getAt(0).fields().get("binary_field").value()).toBytesArray(), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8"))));
}
}