From 28409e45090a74a4d480dfcd34e399f7f39ed06e Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 2 May 2016 09:07:25 -0400 Subject: [PATCH] Add support for .empty to expressions, and some docs improvements Closes #18077 --- .../modules/scripting/scripting.asciidoc | 19 ++++- .../expression/CountMethodValueSource.java | 2 +- .../expression/EmptyMemberValueSource.java | 83 +++++++++++++++++++ .../ExpressionScriptEngineService.java | 26 +++++- .../expression/MoreExpressionTests.java | 26 +++++- 5 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc index 047ad0aa136..5935554a368 100644 --- a/docs/reference/modules/scripting/scripting.asciidoc +++ b/docs/reference/modules/scripting/scripting.asciidoc @@ -455,11 +455,25 @@ for details on what operators and functions are available. Variables in `expression` scripts are available to access: -* Single valued document fields, e.g. `doc['myfield'].value` -* Single valued document fields can also be accessed without `.value` e.g. `doc['myfield']` +* document fields, e.g. `doc['myfield'].value` or just `doc['myfield']`. +* whether the field is empty, e.g. `doc['myfield'].empty` * Parameters passed into the script, e.g. `mymodifier` * The current document's score, `_score` (only available when used in a `script_score`) +When a document is missing the field completely, by default the value will be treated as `0`. +You can treat it as another value instead, e.g. `doc['myfield'].empty ? 100 : doc['myfield'].value` + +When a document has multiple values for the field, by default the minimum value is returned. +You can choose a different value instead, e.g. `doc['myfield'].sum()`. The following methods are available +for any field: + +* min() +* max() +* avg() +* median() +* sum() +* count() + Variables in `expression` scripts that are of type `date` may use the following member methods: * getYear() @@ -477,7 +491,6 @@ There are a few limitations relative to other script languages: * Only numeric fields may be accessed * Stored fields are not available -* If a field is sparse (only some documents contain a value), documents missing the field will have a value of `0` [float] === Score diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java index 043a11eebad..6f397c02bd3 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java @@ -63,7 +63,7 @@ public class CountMethodValueSource extends ValueSource { @Override public int hashCode() { - return fieldData.hashCode(); + return 31 * getClass().hashCode() + fieldData.hashCode(); } @Override diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java new file mode 100644 index 00000000000..b8c101e8abc --- /dev/null +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.docvalues.DoubleDocValues; +import org.elasticsearch.index.fielddata.AtomicNumericFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; + +/** + * ValueSource to return non-zero if a field is missing. + *

+ * This is essentially sugar over !count() + */ +public class EmptyMemberValueSource extends ValueSource { + protected IndexFieldData fieldData; + + protected EmptyMemberValueSource(IndexFieldData fieldData) { + this.fieldData = Objects.requireNonNull(fieldData); + } + + @Override + @SuppressWarnings("rawtypes") // ValueSource uses a rawtype + public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { + AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf); + final SortedNumericDoubleValues values = leafData.getDoubleValues(); + return new DoubleDocValues(this) { + @Override + public double doubleVal(int doc) { + values.setDocument(doc); + if (values.count() == 0) { + return 1; + } else { + return 0; + } + } + }; + } + + @Override + public int hashCode() { + return 31 * getClass().hashCode() + fieldData.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + EmptyMemberValueSource other = (EmptyMemberValueSource) obj; + if (!fieldData.equals(other.fieldData)) return false; + return true; + } + + @Override + public String description() { + return "empty: field(" + fieldData.getFieldName() + ")"; + } +} diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index d78f80bfe49..2bfd2928d57 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -65,6 +65,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements public static final List TYPES = Collections.singletonList(NAME); + // these methods only work on dates, e.g. doc['datefield'].getYear() protected static final String GET_YEAR_METHOD = "getYear"; protected static final String GET_MONTH_METHOD = "getMonth"; protected static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth"; @@ -72,6 +73,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements protected static final String GET_MINUTES_METHOD = "getMinutes"; protected static final String GET_SECONDS_METHOD = "getSeconds"; + // these methods work on any field, e.g. doc['field'].sum() protected static final String MINIMUM_METHOD = "min"; protected static final String MAXIMUM_METHOD = "max"; protected static final String AVERAGE_METHOD = "avg"; @@ -79,6 +81,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements protected static final String SUM_METHOD = "sum"; protected static final String COUNT_METHOD = "count"; + // these variables work on any field, e.g. doc['field'].value + protected static final String VALUE_VARIABLE = "value"; + protected static final String EMPTY_VARIABLE = "empty"; + @Inject public ExpressionScriptEngineService(Settings settings) { super(settings); @@ -169,6 +175,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements } else { String fieldname = null; String methodname = null; + String variablename = VALUE_VARIABLE; // .value is the default for doc['field'], its optional. VariableContext[] parts = VariableContext.parse(variable); if (parts[0].text.equals("doc") == false) { throw new ScriptException("Unknown variable [" + parts[0].text + "] in expression"); @@ -181,8 +188,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements if (parts.length == 3) { if (parts[2].type == VariableContext.Type.METHOD) { methodname = parts[2].text; - } else if (parts[2].type != VariableContext.Type.MEMBER || !"value".equals(parts[2].text)) { - throw new ScriptException("Only the member variable [value] or member methods may be accessed on a field when not accessing the field directly"); + } else if (parts[2].type == VariableContext.Type.MEMBER) { + variablename = parts[2].text; + } else { + throw new ScriptException("Only member variables or member methods may be accessed on a field when not accessing the field directly"); } } if (parts.length > 3) { @@ -201,7 +210,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements throw new ScriptException("Field [" + fieldname + "] used in expression must be numeric"); } if (methodname == null) { - bindings.add(variable, new FieldDataValueSource(fieldData, MultiValueMode.MIN)); + bindings.add(variable, getVariableValueSource(fieldType, fieldData, fieldname, variablename)); } else { bindings.add(variable, getMethodValueSource(fieldType, fieldData, fieldname, methodname)); } @@ -245,6 +254,17 @@ public class ExpressionScriptEngineService extends AbstractComponent implements throw new IllegalArgumentException("Member method [" + methodName + "] does not exist."); } } + + protected ValueSource getVariableValueSource(MappedFieldType fieldType, IndexFieldData fieldData, String fieldName, String memberName) { + switch (memberName) { + case VALUE_VARIABLE: + return new FieldDataValueSource(fieldData, MultiValueMode.MIN); + case EMPTY_VARIABLE: + return new EmptyMemberValueSource(fieldData); + default: + throw new IllegalArgumentException("Member variable [" + memberName + "] does not exist."); + } + } protected ValueSource getDateMethodValueSource(MappedFieldType fieldType, IndexFieldData fieldData, String fieldName, String methodName, int calendarType) { if (fieldType instanceof LegacyDateFieldMapper.DateFieldType == false diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java index b53245fda20..50a9900e3d9 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java @@ -164,10 +164,10 @@ public class MoreExpressionTests extends ESIntegTestCase { } public void testMultiValueMethods() throws Exception { - ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double")); + ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double", "double2", "type=double")); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test", "doc", "1").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double1", "1.2", "double1", "2.4"), + client().prepareIndex("test", "doc", "1").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double1", "1.2", "double1", "2.4", "double2", "3.0"), client().prepareIndex("test", "doc", "2").setSource("double0", "5.0", "double1", "3.0"), client().prepareIndex("test", "doc", "3").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double0", "-1.5", "double1", "4.0")); @@ -227,6 +227,24 @@ public class MoreExpressionTests extends ESIntegTestCase { assertEquals(2.5, hits.getAt(0).field("foo").getValue(), 0.0D); assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D); assertEquals(1.5, hits.getAt(2).field("foo").getValue(), 0.0D); + + // make sure count() works for missing + rsp = buildRequest("doc['double2'].count()").get(); + assertSearchResponse(rsp); + hits = rsp.getHits(); + assertEquals(3, hits.getTotalHits()); + assertEquals(1.0, hits.getAt(0).field("foo").getValue(), 0.0D); + assertEquals(0.0, hits.getAt(1).field("foo").getValue(), 0.0D); + assertEquals(0.0, hits.getAt(2).field("foo").getValue(), 0.0D); + + // make sure .empty works in the same way + rsp = buildRequest("doc['double2'].empty ? 5.0 : 2.0").get(); + assertSearchResponse(rsp); + hits = rsp.getHits(); + assertEquals(3, hits.getTotalHits()); + assertEquals(2.0, hits.getAt(0).field("foo").getValue(), 0.0D); + assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D); + assertEquals(5.0, hits.getAt(2).field("foo").getValue(), 0.0D); } public void testInvalidDateMethodCall() throws Exception { @@ -363,8 +381,8 @@ public class MoreExpressionTests extends ESIntegTestCase { } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ScriptException", e.toString().contains("ScriptException"), equalTo(true)); - assertThat(e.toString() + "should have contained member variable [value] or member methods may be accessed", - e.toString().contains("member variable [value] or member methods may be accessed"), equalTo(true)); + assertThat(e.toString() + "should have contained member variable [bogus] does not exist", + e.toString().contains("Member variable [bogus] does not exist"), equalTo(true)); } }