diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index 5d198520e87..750802c4ec2 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -389,9 +389,23 @@ for details on what operators and functions are available. Variables in `expression` scripts are available to access: * Single valued document fields, e.g. `doc['myfield'].value` +* Single valued document fields can also be accessed without `.value` e.g. `doc['myfield']` * Parameters passed into the script, e.g. `mymodifier` * The current document's score, `_score` (only available when used in a `script_score`) +Variables in `expression` scripts that are of type `date` may use the following member methods: + +* getYear() +* getMonth() +* getDayOfMonth() +* getHourOfDay() +* getMinutes() +* getSeconds() + +The following example shows the difference in years between the `date` fields date0 and date1: + +`doc['date1'].getYear() - doc['date0'].getYear()` + There are a few limitations relative to other script languages: * Only numeric fields may be accessed diff --git a/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java b/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java new file mode 100644 index 00000000000..64eed0741bc --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; + +import org.apache.lucene.queries.function.ValueSource; +import org.elasticsearch.index.fielddata.AtomicNumericFieldData; + +class DateMethodFunctionValues extends FieldDataFunctionValues { + private final int calendarType; + private final Calendar calendar; + + DateMethodFunctionValues(ValueSource parent, AtomicNumericFieldData data, int calendarType) { + super(parent, data); + + this.calendarType = calendarType; + calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT); + } + + @Override + public double doubleVal(int docId) { + long millis = (long)dataAccessor.get(docId); + calendar.setTimeInMillis(millis); + return calendar.get(calendarType); + } +} diff --git a/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java new file mode 100644 index 00000000000..a157790e2bb --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.queries.function.FunctionValues; + +import org.elasticsearch.index.fielddata.AtomicFieldData; +import org.elasticsearch.index.fielddata.AtomicNumericFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; + +class DateMethodValueSource extends FieldDataValueSource { + + protected final String methodName; + protected final int calendarType; + + DateMethodValueSource(IndexFieldData indexFieldData, String methodName, int calendarType) { + super(indexFieldData); + + Objects.requireNonNull(methodName); + + this.methodName = methodName; + this.calendarType = calendarType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + + DateMethodValueSource that = (DateMethodValueSource) o; + + if (calendarType != that.calendarType) return false; + return methodName.equals(that.methodName); + + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + methodName.hashCode(); + result = 31 * result + calendarType; + return result; + } + + @Override + public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { + AtomicFieldData leafData = fieldData.load(leaf); + assert(leafData instanceof AtomicNumericFieldData); + + return new DateMethodFunctionValues(this, (AtomicNumericFieldData)leafData, calendarType); + } + + @Override + public String description() { + return methodName + ": field(" + fieldData.getFieldNames().toString() + ")"; + } +} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 23841942104..6d6f986432b 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -23,6 +23,7 @@ import org.apache.lucene.expressions.Expression; import org.apache.lucene.expressions.SimpleBindings; import org.apache.lucene.expressions.js.JavascriptCompiler; import org.apache.lucene.expressions.js.VariableContext; +import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; import org.apache.lucene.search.SortField; import org.elasticsearch.common.Nullable; @@ -32,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.index.mapper.core.NumberFieldMapper; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; @@ -40,6 +42,7 @@ import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.lookup.SearchLookup; import java.text.ParseException; +import java.util.Calendar; import java.util.Map; /** @@ -50,6 +53,13 @@ public class ExpressionScriptEngineService extends AbstractComponent implements public static final String NAME = "expression"; + protected static final String GET_YEAR_METHOD = "getYear"; + protected static final String GET_MONTH_METHOD = "getMonth"; + protected static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth"; + protected static final String GET_HOUR_OF_DAY_METHOD = "getHourOfDay"; + protected static final String GET_MINUTES_METHOD = "getMinutes"; + protected static final String GET_SECONDS_METHOD = "getSeconds"; + @Inject public ExpressionScriptEngineService(Settings settings) { super(settings); @@ -112,19 +122,30 @@ public class ExpressionScriptEngineService extends AbstractComponent implements } } else { + String fieldname = null; + String methodname = null; VariableContext[] parts = VariableContext.parse(variable); if (parts[0].text.equals("doc") == false) { throw new ExpressionScriptCompilationException("Unknown variable [" + parts[0].text + "] in expression"); } if (parts.length < 2 || parts[1].type != VariableContext.Type.STR_INDEX) { - throw new ExpressionScriptCompilationException("Variable 'doc' in expression must be used with a specific field like: doc['myfield'].value"); + throw new ExpressionScriptCompilationException("Variable 'doc' in expression must be used with a specific field like: doc['myfield']"); + } else { + fieldname = parts[1].text; } - if (parts.length < 3 || parts[2].type != VariableContext.Type.MEMBER || parts[2].text.equals("value") == false) { - throw new ExpressionScriptCompilationException("Invalid member for field data in expression. Only '.value' is currently supported."); + if (parts.length == 3) { + if (parts[2].type == VariableContext.Type.METHOD) { + methodname = parts[2].text; + } else if (parts[2].type != VariableContext.Type.MEMBER || !"value".equals(parts[2].text)) { + throw new ExpressionScriptCompilationException("Only the member variable [value] or member methods may be accessed on a field when not accessing the field directly"); + } + } + if (parts.length > 3) { + throw new ExpressionScriptCompilationException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()"); } - String fieldname = parts[1].text; FieldMapper field = mapper.smartNameFieldMapper(fieldname); + if (field == null) { throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression does not exist in mappings"); } @@ -132,14 +153,46 @@ public class ExpressionScriptEngineService extends AbstractComponent implements // TODO: more context (which expression?) throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression must be numeric"); } + IndexFieldData fieldData = lookup.doc().fieldDataService().getForField((NumberFieldMapper)field); - bindings.add(variable, new FieldDataValueSource(fieldData)); + if (methodname == null) { + bindings.add(variable, new FieldDataValueSource(fieldData)); + } else { + bindings.add(variable, getMethodValueSource(field, fieldData, fieldname, methodname)); + } } } return new ExpressionScript((Expression)compiledScript, bindings, specialValue); } + protected ValueSource getMethodValueSource(FieldMapper field, IndexFieldData fieldData, String fieldName, String methodName) { + switch (methodName) { + case GET_YEAR_METHOD: + return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.YEAR); + case GET_MONTH_METHOD: + return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.MONTH); + case GET_DAY_OF_MONTH_METHOD: + return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.DAY_OF_MONTH); + case GET_HOUR_OF_DAY_METHOD: + return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.HOUR_OF_DAY); + case GET_MINUTES_METHOD: + return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.MINUTE); + case GET_SECONDS_METHOD: + return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.SECOND); + default: + throw new IllegalArgumentException("Member method [" + methodName + "] does not exist."); + } + } + + protected ValueSource getDateMethodValueSource(FieldMapper field, IndexFieldData fieldData, String fieldName, String methodName, int calendarType) { + if (!(field instanceof DateFieldMapper)) { + throw new IllegalArgumentException("Member method [" + methodName + "] can only be used with a date field type, not the field [" + fieldName + "]."); + } + + return new DateMethodValueSource(fieldData, methodName, calendarType); + } + @Override public ExecutableScript executable(Object compiledScript, @Nullable Map vars) { throw new UnsupportedOperationException("Cannot use expressions for updates"); diff --git a/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java index 16e3d35bb61..7a97532068a 100644 --- a/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java +++ b/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java @@ -19,7 +19,6 @@ package org.elasticsearch.script.expression; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; @@ -29,15 +28,18 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import java.io.IOException; import java.util.Map; +import java.util.Objects; /** * A {@link ValueSource} wrapper for field data. */ class FieldDataValueSource extends ValueSource { - IndexFieldData fieldData; + protected IndexFieldData fieldData; + + protected FieldDataValueSource(IndexFieldData d) { + Objects.requireNonNull(d); - FieldDataValueSource(IndexFieldData d) { fieldData = d; } @@ -49,8 +51,13 @@ class FieldDataValueSource extends ValueSource { } @Override - public boolean equals(Object other) { - return fieldData.equals(other); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FieldDataValueSource that = (FieldDataValueSource) o; + + return fieldData.equals(that.fieldData); } @Override diff --git a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java index 8ee8d1dcbf1..1f04063a42d 100644 --- a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java +++ b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -61,6 +61,15 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { } public void testBasic() throws Exception { + createIndex("test"); + ensureGreen("test"); + client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get(); + SearchResponse rsp = buildRequest("doc['foo'] + 1").get(); + assertEquals(1, rsp.getHits().getTotalHits()); + assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue()); + } + + public void testBasicUsingDotValue() throws Exception { createIndex("test"); ensureGreen("test"); client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get(); @@ -89,13 +98,56 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { assertEquals("2", hits.getAt(2).getId()); } + public void testDateMethods() throws Exception { + ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date")); + ensureGreen("test"); + indexRandom(true, + client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), + client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); + SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get(); + assertEquals(2, rsp.getHits().getTotalHits()); + SearchHits hits = rsp.getHits(); + assertEquals(5.0, hits.getAt(0).field("foo").getValue()); + assertEquals(-11.0, hits.getAt(1).field("foo").getValue()); + rsp = buildRequest("doc['date0'].getHourOfDay() + doc['date1'].getDayOfMonth()").get(); + assertEquals(2, rsp.getHits().getTotalHits()); + hits = rsp.getHits(); + assertEquals(5.0, hits.getAt(0).field("foo").getValue()); + assertEquals(24.0, hits.getAt(1).field("foo").getValue()); + rsp = buildRequest("doc['date1'].getMonth() + 1").get(); + assertEquals(2, rsp.getHits().getTotalHits()); + hits = rsp.getHits(); + assertEquals(9.0, hits.getAt(0).field("foo").getValue()); + assertEquals(10.0, hits.getAt(1).field("foo").getValue()); + rsp = buildRequest("doc['date1'].getYear()").get(); + assertEquals(2, rsp.getHits().getTotalHits()); + hits = rsp.getHits(); + assertEquals(1985.0, hits.getAt(0).field("foo").getValue()); + assertEquals(1983.0, hits.getAt(1).field("foo").getValue()); + } + + public void testInvalidDateMethodCall() throws Exception { + ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double", "type=double")); + ensureGreen("test"); + indexRandom(true, client().prepareIndex("test", "doc", "1").setSource("double", "178000000.0")); + try { + buildRequest("doc['double'].getYear()").get(); + fail(); + } catch (SearchPhaseExecutionException e) { + assertThat(e.toString() + "should have contained IllegalArgumentException", + e.toString().contains("IllegalArgumentException"), equalTo(true)); + assertThat(e.toString() + "should have contained can only be used with a date field type", + e.toString().contains("can only be used with a date field type"), equalTo(true)); + } + } + public void testSparseField() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long")); ensureGreen("test"); indexRandom(true, client().prepareIndex("test", "doc", "1").setSource("x", 4), client().prepareIndex("test", "doc", "2").setSource("y", 2)); - SearchResponse rsp = buildRequest("doc['x'].value + 1").get(); + SearchResponse rsp = buildRequest("doc['x'] + 1").get(); ElasticsearchAssertions.assertSearchResponse(rsp); SearchHits hits = rsp.getHits(); assertEquals(2, rsp.getHits().getTotalHits()); @@ -108,7 +160,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { ensureGreen("test"); client().prepareIndex("test", "doc", "1").setSource("x", 4).setRefresh(true).get(); try { - buildRequest("doc['bogus'].value").get(); + buildRequest("doc['bogus']").get(); fail("Expected missing field to cause failure"); } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ExpressionScriptCompilationException", @@ -126,7 +178,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { client().prepareIndex("test", "doc", "2").setSource("x", 3), client().prepareIndex("test", "doc", "3").setSource("x", 5)); // a = int, b = double, c = long - String script = "doc['x'].value * a + b + ((c + doc['x'].value) > 5000000009 ? 1 : 0)"; + String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)"; SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get(); SearchHits hits = rsp.getHits(); assertEquals(3, hits.getTotalHits()); @@ -164,7 +216,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { public void testNonNumericField() { client().prepareIndex("test", "doc", "1").setSource("text", "this is not a number").setRefresh(true).get(); try { - buildRequest("doc['text'].value").get(); + buildRequest("doc['text']").get(); fail("Expected text field to cause execution failure"); } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ExpressionScriptCompilationException", @@ -208,8 +260,8 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should have contained ExpressionScriptCompilationException", e.toString().contains("ExpressionScriptCompilationException"), equalTo(true)); - assertThat(e.toString() + "should have contained field member error", - e.toString().contains("Invalid member for field"), equalTo(true)); + assertThat(e.toString() + "should have contained member variable [value] or member methods may be accessed", + e.toString().contains("member variable [value] or member methods may be accessed"), equalTo(true)); } }