From f37db2fe17e73c1772ecb9e263ae9f5535c917bc Mon Sep 17 00:00:00 2001 From: umeshdangat Date: Tue, 22 Nov 2016 13:23:23 -0800 Subject: [PATCH] Support binary field type in script values (#21484) Add ScriptDocValues.BytesRefs for reading binary fieldtype --- .../index/fielddata/ScriptDocValues.java | 43 ++++++++++- .../plain/BytesBinaryDVAtomicFieldData.java | 2 +- .../scriptfilter/ScriptQuerySearchIT.java | 77 +++++++++++++++++-- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 403a1290546..308fd7dd0a7 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -25,7 +25,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.unit.DistanceUnit; import org.joda.time.DateTimeZone; import org.joda.time.MutableDateTime; import org.joda.time.ReadableDateTime; @@ -325,4 +324,46 @@ public interface ScriptDocValues extends List { } } + + public static class BytesRefs extends AbstractList implements ScriptDocValues { + + private final SortedBinaryDocValues values; + + public BytesRefs(SortedBinaryDocValues values) { + this.values = values; + } + + @Override + public void setNextDocId(int docId) { + values.setDocument(docId); + } + + public SortedBinaryDocValues getInternalValues() { + return this.values; + } + + public BytesRef getValue() { + int numValues = values.count(); + if (numValues == 0) { + return new BytesRef(); + } + return values.valueAt(0); + } + + @Override + public List getValues() { + return Collections.unmodifiableList(this); + } + + @Override + public BytesRef get(int index) { + return values.valueAt(index); + } + + @Override + public int size() { + return values.count(); + } + } + } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java index 9963f7d51a8..8d43241ba75 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java @@ -101,7 +101,7 @@ final class BytesBinaryDVAtomicFieldData implements AtomicFieldData { @Override public ScriptDocValues getScriptValues() { - throw new UnsupportedOperationException(); + return new ScriptDocValues.BytesRefs(getBytesValues()); } @Override diff --git a/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java index 02fde4d9717..b7eb964a4ee 100644 --- a/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/scriptfilter/ScriptQuerySearchIT.java @@ -21,6 +21,8 @@ package org.elasticsearch.search.scriptfilter; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; @@ -30,18 +32,22 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; +import java.io.IOException; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -@ESIntegTestCase.ClusterScope(scope= ESIntegTestCase.Scope.SUITE) +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) public class ScriptQuerySearchIT extends ESIntegTestCase { @Override @@ -74,6 +80,16 @@ public class ScriptQuerySearchIT extends ESIntegTestCase { return num1.getValue() > param1; }); + scripts.put("doc['binaryData'].get(0).length", vars -> { + Map doc = (Map) vars.get("doc"); + return ((ScriptDocValues.BytesRefs) doc.get("binaryData")).get(0).length; + }); + + scripts.put("doc['binaryData'].get(0).length > 15", vars -> { + Map doc = (Map) vars.get("doc"); + return ((ScriptDocValues.BytesRefs) doc.get("binaryData")).get(0).length > 15; + }); + return scripts; } } @@ -87,6 +103,57 @@ public class ScriptQuerySearchIT extends ESIntegTestCase { .build(); } + public void testCustomScriptBinaryField() throws Exception { + final byte[] randomBytesDoc1 = getRandomBytes(15); + final byte[] randomBytesDoc2 = getRandomBytes(16); + + assertAcked( + client().admin().indices().prepareCreate("my-index") + .addMapping("my-type", createMappingSource("binary")) + .setSettings(indexSettings()) + ); + client().prepareIndex("my-index", "my-type", "1") + .setSource(jsonBuilder().startObject().field("binaryData", + Base64.getEncoder().encodeToString(randomBytesDoc1)).endObject()) + .get(); + flush(); + client().prepareIndex("my-index", "my-type", "2") + .setSource(jsonBuilder().startObject().field("binaryData", + Base64.getEncoder().encodeToString(randomBytesDoc2)).endObject()) + .get(); + flush(); + refresh(); + + SearchResponse response = client().prepareSearch() + .setQuery(scriptQuery( + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['binaryData'].get(0).length > 15", Collections.emptyMap()))) + .addScriptField("sbinaryData", + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['binaryData'].get(0).length", Collections.emptyMap())) + .get(); + + assertThat(response.getHits().totalHits(), equalTo(1L)); + assertThat(response.getHits().getAt(0).id(), equalTo("2")); + assertThat(response.getHits().getAt(0).fields().get("sbinaryData").values().get(0), equalTo(16)); + + } + + private byte[] getRandomBytes(int len) { + final byte[] randomBytes = new byte[len]; + new Random().nextBytes(randomBytes); + return randomBytes; + } + + private XContentBuilder createMappingSource(String fieldType) throws IOException { + return XContentFactory.jsonBuilder().startObject().startObject("my-type") + .startObject("properties") + .startObject("binaryData") + .field("type", fieldType) + .field("doc_values", "true") + .endObject() + .endObject() + .endObject().endObject(); + } + public void testCustomScriptBoost() throws Exception { createIndex("test"); client().prepareIndex("test", "type1", "1") @@ -105,10 +172,10 @@ public class ScriptQuerySearchIT extends ESIntegTestCase { logger.info("running doc['num1'].value > 1"); SearchResponse response = client().prepareSearch() .setQuery(scriptQuery( - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > 1", Collections.emptyMap()))) + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > 1", Collections.emptyMap()))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())) + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())) .get(); assertThat(response.getHits().totalHits(), equalTo(2L)); @@ -126,7 +193,7 @@ public class ScriptQuerySearchIT extends ESIntegTestCase { .setQuery(scriptQuery(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > param1", params))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())) + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())) .get(); assertThat(response.getHits().totalHits(), equalTo(1L)); @@ -141,7 +208,7 @@ public class ScriptQuerySearchIT extends ESIntegTestCase { .setQuery(scriptQuery(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value > param1", params))) .addSort("num1", SortOrder.ASC) .addScriptField("sNum1", - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())) + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['num1'].value", Collections.emptyMap())) .get(); assertThat(response.getHits().totalHits(), equalTo(3L));