honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a doc value format to binary fields. (#30860) 

This will be necessary for the `docvalue_fields` option to work correctly once
we use the field's doc-value format to format doc-value fields. Binary values
are formatted as base64-encoded strings.
This commit is contained in:
Adrien Grand 2018-06-05 12:42:13 +02:00 committed by GitHub
parent 0fad7cc99a
commit f2892f1bed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 77 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
View File

@ -40,6 +40,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.DocValueFormat;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.Base64;
@ -104,6 +106,10 @@ public class BinaryFieldMapper extends FieldMapper {
return CONTENT_TYPE;
}
@Override
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
return DocValueFormat.BINARY;
}
@Override
public BytesReference valueForDisplay(Object value) {

45
server/src/main/java/org/elasticsearch/search/DocValueFormat.java
View File

@ -39,6 +39,7 @@ import java.text.DecimalFormatSymbols;
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Base64;
import java.util.Locale;
import java.util.Objects;
import java.util.function.LongSupplier;
@ -121,6 +122,50 @@ public interface DocValueFormat extends NamedWriteable {
}
};
DocValueFormat BINARY = new DocValueFormat() {
@Override
public String getWriteableName() {
return "binary";
}
@Override
public void writeTo(StreamOutput out) throws IOException {
}
@Override
public Object format(long value) {
throw new UnsupportedOperationException();
}
@Override
public Object format(double value) {
throw new UnsupportedOperationException();
}
@Override
public String format(BytesRef value) {
return Base64.getEncoder()
.withoutPadding()
.encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length));
}
@Override
public long parseLong(String value, boolean roundUp, LongSupplier now) {
throw new UnsupportedOperationException();
}
@Override
public double parseDouble(String value, boolean roundUp, LongSupplier now) {
throw new UnsupportedOperationException();
}
@Override
public BytesRef parseBytesRef(String value) {
return new BytesRef(Base64.getDecoder().decode(value));
}
};
final class DateTime implements DocValueFormat {
public static final String NAME = "date_time";

1
server/src/main/java/org/elasticsearch/search/SearchModule.java
View File

@ -645,6 +645,7 @@ public class SearchModule {
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);
}
/**

14
server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
View File

@ -44,6 +44,7 @@ public class DocValueFormatTests extends ESTestCase {
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY));
NamedWriteableRegistry registry = new NamedWriteableRegistry(entries);
BytesStreamOutput out = new BytesStreamOutput();
@ -82,6 +83,11 @@ public class DocValueFormatTests extends ESTestCase {
out.writeNamedWriteable(DocValueFormat.RAW);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));
out = new BytesStreamOutput();
out.writeNamedWriteable(DocValueFormat.BINARY);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.BINARY, in.readNamedWriteable(DocValueFormat.class));
}
public void testRawFormat() {
@ -96,6 +102,14 @@ public class DocValueFormatTests extends ESTestCase {
assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc")));
}
public void testBinaryFormat() {
assertEquals("", DocValueFormat.BINARY.format(new BytesRef()));
assertEquals("KmQ", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100})));
assertEquals(new BytesRef(), DocValueFormat.BINARY.parseBytesRef(""));
assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ"));
}
public void testBooleanFormat() {
assertEquals(false, DocValueFormat.BOOLEAN.format(0));
assertEquals(true, DocValueFormat.BOOLEAN.format(1));

14
server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.fields;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@ -700,7 +701,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(fields.get("test_field").getValue(), equalTo("foobar"));
}
public void testFieldsPulledFromFieldData() throws Exception {
public void testDocValueFields() throws Exception {
createIndex("test");
String mapping = Strings
@ -744,6 +745,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.endObject()
.startObject("binary_field")
.field("type", "binary")
.field("doc_values", true) // off by default on binary fields
.endObject()
.startObject("ip_field")
.field("type", "ip")
@ -766,6 +768,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.field("double_field", 6.0d)
.field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
.field("boolean_field", true)
.field("binary_field", new byte[] {42, 100})
.field("ip_field", "::1")
.endObject()).execute().actionGet();
@ -782,6 +785,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.addDocValueField("double_field")
.addDocValueField("date_field")
.addDocValueField("boolean_field")
.addDocValueField("binary_field")
.addDocValueField("ip_field");
SearchResponse searchResponse = builder.execute().actionGet();
@ -790,7 +794,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
Set<String> fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
"ip_field")));
"binary_field", "ip_field")));
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@ -802,6 +806,8 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(),
equalTo(new BytesRef(new byte[] {42, 100})));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
builder = client().prepareSearch().setQuery(matchAllQuery())
@ -815,6 +821,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.addDocValueField("double_field", "use_field_mapping")
.addDocValueField("date_field", "use_field_mapping")
.addDocValueField("boolean_field", "use_field_mapping")
.addDocValueField("binary_field", "use_field_mapping")
.addDocValueField("ip_field", "use_field_mapping");
searchResponse = builder.execute().actionGet();
@ -823,7 +830,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
"ip_field")));
"binary_field", "ip_field")));
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@ -836,6 +843,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
builder = client().prepareSearch().setQuery(matchAllQuery())