From baed02bbe2717472b3502ac486fcc26b91e63a95 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 12 Jan 2017 15:26:09 -0500 Subject: [PATCH] Whitelist some ScriptDocValues in painless (#22600) Without this whitelist painless can't use ip or binary doc values. Closes #22584 --- .../index/mapper/IpFieldMapper.java | 7 +- .../elasticsearch/painless/Definition.java | 6 +- .../painless/org.elasticsearch.txt | 20 + .../test/painless/50_script_doc_values.yaml | 368 ++++++++++++++++++ 4 files changed, 392 insertions(+), 9 deletions(-) create mode 100644 modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index a892fbe2214..0b3b4fd8a40 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; @@ -32,7 +31,6 @@ import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; @@ -41,7 +39,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; @@ -51,11 +48,9 @@ import java.io.IOException; import java.net.InetAddress; import java.util.AbstractList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; /** A {@link FieldMapper} for ip addresses. */ @@ -238,7 +233,7 @@ public class IpFieldMapper extends FieldMapper { InetAddressPoint.decode(min), InetAddressPoint.decode(max)); } - private static class IpScriptDocValues extends AbstractList implements ScriptDocValues { + public static final class IpScriptDocValues extends AbstractList implements ScriptDocValues { private final RandomAccessOrds values; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java index e0bdfbf17c2..bde7f27ba26 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java @@ -566,11 +566,11 @@ public final class Definition { } if (line.startsWith("class ")) { String elements[] = line.split("\u0020"); - assert elements[2].equals("->"); + assert elements[2].equals("->") : "Invalid struct definition [" + String.join(" ", elements) +"]"; if (elements.length == 7) { hierarchy.put(elements[1], Arrays.asList(elements[5].split(","))); } else { - assert elements.length == 5; + assert elements.length == 5 : "Invalid struct definition [" + String.join(" ", elements) + "]"; } String className = elements[1]; String javaPeer = elements[3]; @@ -612,7 +612,7 @@ public final class Definition { } } } catch (Exception e) { - throw new RuntimeException("syntax error in " + file + ", line: " + currentLine, e); + throw new RuntimeException("error in " + file + ", line: " + currentLine, e); } } return hierarchy; diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt index bd2f1c1b5db..92eccc5ac3c 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt @@ -114,6 +114,26 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Booleans -> org.elastics List getValues() } +class org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs -> org.elasticsearch.index.fielddata.ScriptDocValues$BytesRefs extends List,Collection,Iterable,Object { + BytesRef get(int) + BytesRef getValue() + List getValues() +} + +class BytesRef -> org.apache.lucene.util.BytesRef extends Object { + byte[] bytes + int offset + int length + boolean bytesEquals(BytesRef) + String utf8ToString() +} + +class org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues -> org.elasticsearch.index.mapper.IpFieldMapper$IpFieldType$IpScriptDocValues extends List,Collection,Iterable,Object { + String get(int) + String getValue() + List getValues() +} + # for testing. # currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object { diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml new file mode 100644 index 00000000000..30cc9f7f479 --- /dev/null +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml @@ -0,0 +1,368 @@ +setup: + - do: + indices.create: + index: test + body: + mappings: + test: + properties: + binary: + type: binary + doc_values: true + boolean: + type: boolean + date: + type: date + geo_point: + type: geo_point + ip: + type: ip + keyword: + type: keyword + long: + type: long + integer: + type: integer + short: + type: short + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + scaled_float: + type: scaled_float + scaling_factor: 100 + token_count: + type: token_count + analyzer: standard + + - do: + index: + index: test + type: test + id: 1 + body: + binary: U29tZSBiaW5hcnkgYmxvYg== + boolean: true + date: 2017-01-01T12:11:12 + geo_point: 41.12,-71.34 + ip: 192.168.0.1 + keyword: not split at all + long: 12348732141234 + integer: 134134566 + short: 1324 + byte: 12 + double: 3.14159265358979 + float: 3.141592654 + half_float: 3.140625 + scaled_float: 3.14 + token_count: count all these words please + + - do: + indices.refresh: {} + +--- +"binary": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['binary'].get(0).utf8ToString()" + - match: { hits.hits.0.fields.field.0: "Some binary blob" } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['binary'].value.utf8ToString()" + - match: { hits.hits.0.fields.field.0: "Some binary blob" } + +--- +"boolean": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['boolean'].get(0)" + - match: { hits.hits.0.fields.field.0: true } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['boolean'].value" + - match: { hits.hits.0.fields.field.0: true } + +--- +"date": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['date'].get(0)" + - match: { hits.hits.0.fields.field.0: 1483272672000 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['date'].value" + - match: { hits.hits.0.fields.field.0: 1483272672000 } + +--- +"geo_point": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['geo_point'].get(0)" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['geo_point'].value" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + +--- +"ip": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['ip'].get(0)" + - match: { hits.hits.0.fields.field.0: "192.168.0.1" } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['ip'].value" + - match: { hits.hits.0.fields.field.0: "192.168.0.1" } + +--- +"keyword": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['keyword'].get(0)" + - match: { hits.hits.0.fields.field.0: "not split at all" } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['keyword'].value" + - match: { hits.hits.0.fields.field.0: "not split at all" } + +--- +"long": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['long'].get(0)" + - match: { hits.hits.0.fields.field.0: 12348732141234 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['long'].value" + - match: { hits.hits.0.fields.field.0: 12348732141234 } + +--- +"integer": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['integer'].get(0)" + - match: { hits.hits.0.fields.field.0: 134134566 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['integer'].value" + - match: { hits.hits.0.fields.field.0: 134134566 } + +--- +"short": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['short'].get(0)" + - match: { hits.hits.0.fields.field.0: 1324 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['short'].value" + - match: { hits.hits.0.fields.field.0: 1324 } + +--- +"byte": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['byte'].get(0)" + - match: { hits.hits.0.fields.field.0: 12 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['byte'].value" + - match: { hits.hits.0.fields.field.0: 12 } + +--- +"double": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['double'].get(0)" + - match: { hits.hits.0.fields.field.0: 3.14159265358979 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['double'].value" + - match: { hits.hits.0.fields.field.0: 3.14159265358979 } + +--- +"float": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['float'].get(0)" + - match: { hits.hits.0.fields.field.0: 3.1415927410125732 } # this ends up as a double + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['float'].value" + - match: { hits.hits.0.fields.field.0: 3.1415927410125732 } # this ends up as a double + +--- +"half_float": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['half_float'].get(0)" + - match: { hits.hits.0.fields.field.0: 3.140625 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['half_float'].value" + - match: { hits.hits.0.fields.field.0: 3.140625 } + +--- +"scaled_float": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['scaled_float'].get(0)" + - match: { hits.hits.0.fields.field.0: 3.14 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['scaled_float'].value" + - match: { hits.hits.0.fields.field.0: 3.14 } + +--- +"token_count": + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['token_count'].get(0)" + - match: { hits.hits.0.fields.field.0: 5 } + + - do: + search: + body: + script_fields: + field: + script: + inline: "doc['token_count'].value" + - match: { hits.hits.0.fields.field.0: 5 }