Scripting: Add Field Methods
Added infrastructure to allow basic member methods in the expressions language to be called. The methods must have a signature with no arguments. Also added the following member methods for date fields (and it should be easy to add more) * getYear * getMonth * getDayOfMonth * getHourOfDay * getMinutes * getSeconds Allow fields to be accessed without using the member variable [value]. (Note that both ways can be used to access fields for back-compat.) closes #10890
This commit is contained in:
Jack Conradson
parent
d2b12e4fc2
commit
aa968f6b65
|
|
@ -389,9 +389,23 @@ for details on what operators and functions are available.
|
|||
Variables in `expression` scripts are available to access:
|
||||
|
||||
* Single valued document fields, e.g. `doc['myfield'].value`
|
||||
* Single valued document fields can also be accessed without `.value` e.g. `doc['myfield']`
|
||||
* Parameters passed into the script, e.g. `mymodifier`
|
||||
* The current document's score, `_score` (only available when used in a `script_score`)
|
||||
|
||||
Variables in `expression` scripts that are of type `date` may use the following member methods:
|
||||
|
||||
* getYear()
|
||||
* getMonth()
|
||||
* getDayOfMonth()
|
||||
* getHourOfDay()
|
||||
* getMinutes()
|
||||
* getSeconds()
|
||||
|
||||
The following example shows the difference in years between the `date` fields date0 and date1:
|
||||
|
||||
`doc['date1'].getYear() - doc['date0'].getYear()`
|
||||
|
||||
There are a few limitations relative to other script languages:
|
||||
|
||||
* Only numeric fields may be accessed
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.script.expression;
|
||||
|
||||
import java.util.Calendar;
|
||||
import java.util.Locale;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
|
||||
|
||||
class DateMethodFunctionValues extends FieldDataFunctionValues {
|
||||
private final int calendarType;
|
||||
private final Calendar calendar;
|
||||
|
||||
DateMethodFunctionValues(ValueSource parent, AtomicNumericFieldData data, int calendarType) {
|
||||
super(parent, data);
|
||||
|
||||
this.calendarType = calendarType;
|
||||
calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public double doubleVal(int docId) {
|
||||
long millis = (long)dataAccessor.get(docId);
|
||||
calendar.setTimeInMillis(millis);
|
||||
return calendar.get(calendarType);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.script.expression;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.queries.function.FunctionValues;
|
||||
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
|
||||
class DateMethodValueSource extends FieldDataValueSource {
|
||||
|
||||
protected final String methodName;
|
||||
protected final int calendarType;
|
||||
|
||||
DateMethodValueSource(IndexFieldData<?> indexFieldData, String methodName, int calendarType) {
|
||||
super(indexFieldData);
|
||||
|
||||
Objects.requireNonNull(methodName);
|
||||
|
||||
this.methodName = methodName;
|
||||
this.calendarType = calendarType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (!super.equals(o)) return false;
|
||||
|
||||
DateMethodValueSource that = (DateMethodValueSource) o;
|
||||
|
||||
if (calendarType != that.calendarType) return false;
|
||||
return methodName.equals(that.methodName);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = super.hashCode();
|
||||
result = 31 * result + methodName.hashCode();
|
||||
result = 31 * result + calendarType;
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
|
||||
AtomicFieldData leafData = fieldData.load(leaf);
|
||||
assert(leafData instanceof AtomicNumericFieldData);
|
||||
|
||||
return new DateMethodFunctionValues(this, (AtomicNumericFieldData)leafData, calendarType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String description() {
|
||||
return methodName + ": field(" + fieldData.getFieldNames().toString() + ")";
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ import org.apache.lucene.expressions.Expression;
|
|||
import org.apache.lucene.expressions.SimpleBindings;
|
||||
import org.apache.lucene.expressions.js.JavascriptCompiler;
|
||||
import org.apache.lucene.expressions.js.VariableContext;
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
|
|
@ -32,6 +33,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.index.mapper.core.DateFieldMapper;
|
||||
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
|
||||
import org.elasticsearch.script.CompiledScript;
|
||||
import org.elasticsearch.script.ExecutableScript;
|
||||
|
|
@ -40,6 +42,7 @@ import org.elasticsearch.script.SearchScript;
|
|||
import org.elasticsearch.search.lookup.SearchLookup;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.util.Calendar;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
|
|
@ -50,6 +53,13 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
|
|||
|
||||
public static final String NAME = "expression";
|
||||
|
||||
protected static final String GET_YEAR_METHOD = "getYear";
|
||||
protected static final String GET_MONTH_METHOD = "getMonth";
|
||||
protected static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth";
|
||||
protected static final String GET_HOUR_OF_DAY_METHOD = "getHourOfDay";
|
||||
protected static final String GET_MINUTES_METHOD = "getMinutes";
|
||||
protected static final String GET_SECONDS_METHOD = "getSeconds";
|
||||
|
||||
@Inject
|
||||
public ExpressionScriptEngineService(Settings settings) {
|
||||
super(settings);
|
||||
|
|
@ -112,19 +122,30 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
|
|||
}
|
||||
|
||||
} else {
|
||||
String fieldname = null;
|
||||
String methodname = null;
|
||||
VariableContext[] parts = VariableContext.parse(variable);
|
||||
if (parts[0].text.equals("doc") == false) {
|
||||
throw new ExpressionScriptCompilationException("Unknown variable [" + parts[0].text + "] in expression");
|
||||
}
|
||||
if (parts.length < 2 || parts[1].type != VariableContext.Type.STR_INDEX) {
|
||||
throw new ExpressionScriptCompilationException("Variable 'doc' in expression must be used with a specific field like: doc['myfield'].value");
|
||||
throw new ExpressionScriptCompilationException("Variable 'doc' in expression must be used with a specific field like: doc['myfield']");
|
||||
} else {
|
||||
fieldname = parts[1].text;
|
||||
}
|
||||
if (parts.length < 3 || parts[2].type != VariableContext.Type.MEMBER || parts[2].text.equals("value") == false) {
|
||||
throw new ExpressionScriptCompilationException("Invalid member for field data in expression. Only '.value' is currently supported.");
|
||||
if (parts.length == 3) {
|
||||
if (parts[2].type == VariableContext.Type.METHOD) {
|
||||
methodname = parts[2].text;
|
||||
} else if (parts[2].type != VariableContext.Type.MEMBER || !"value".equals(parts[2].text)) {
|
||||
throw new ExpressionScriptCompilationException("Only the member variable [value] or member methods may be accessed on a field when not accessing the field directly");
|
||||
}
|
||||
}
|
||||
if (parts.length > 3) {
|
||||
throw new ExpressionScriptCompilationException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
|
||||
}
|
||||
String fieldname = parts[1].text;
|
||||
|
||||
FieldMapper<?> field = mapper.smartNameFieldMapper(fieldname);
|
||||
|
||||
if (field == null) {
|
||||
throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression does not exist in mappings");
|
||||
}
|
||||
|
|
@ -132,14 +153,46 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
|
|||
// TODO: more context (which expression?)
|
||||
throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression must be numeric");
|
||||
}
|
||||
|
||||
IndexFieldData<?> fieldData = lookup.doc().fieldDataService().getForField((NumberFieldMapper)field);
|
||||
if (methodname == null) {
|
||||
bindings.add(variable, new FieldDataValueSource(fieldData));
|
||||
} else {
|
||||
bindings.add(variable, getMethodValueSource(field, fieldData, fieldname, methodname));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new ExpressionScript((Expression)compiledScript, bindings, specialValue);
|
||||
}
|
||||
|
||||
protected ValueSource getMethodValueSource(FieldMapper<?> field, IndexFieldData<?> fieldData, String fieldName, String methodName) {
|
||||
switch (methodName) {
|
||||
case GET_YEAR_METHOD:
|
||||
return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.YEAR);
|
||||
case GET_MONTH_METHOD:
|
||||
return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.MONTH);
|
||||
case GET_DAY_OF_MONTH_METHOD:
|
||||
return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.DAY_OF_MONTH);
|
||||
case GET_HOUR_OF_DAY_METHOD:
|
||||
return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.HOUR_OF_DAY);
|
||||
case GET_MINUTES_METHOD:
|
||||
return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.MINUTE);
|
||||
case GET_SECONDS_METHOD:
|
||||
return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.SECOND);
|
||||
default:
|
||||
throw new IllegalArgumentException("Member method [" + methodName + "] does not exist.");
|
||||
}
|
||||
}
|
||||
|
||||
protected ValueSource getDateMethodValueSource(FieldMapper<?> field, IndexFieldData<?> fieldData, String fieldName, String methodName, int calendarType) {
|
||||
if (!(field instanceof DateFieldMapper)) {
|
||||
throw new IllegalArgumentException("Member method [" + methodName + "] can only be used with a date field type, not the field [" + fieldName + "].");
|
||||
}
|
||||
|
||||
return new DateMethodValueSource(fieldData, methodName, calendarType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExecutableScript executable(Object compiledScript, @Nullable Map<String, Object> vars) {
|
||||
throw new UnsupportedOperationException("Cannot use expressions for updates");
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
|
||||
package org.elasticsearch.script.expression;
|
||||
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.queries.function.FunctionValues;
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
|
|
@ -29,15 +28,18 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* A {@link ValueSource} wrapper for field data.
|
||||
*/
|
||||
class FieldDataValueSource extends ValueSource {
|
||||
|
||||
IndexFieldData<?> fieldData;
|
||||
protected IndexFieldData<?> fieldData;
|
||||
|
||||
protected FieldDataValueSource(IndexFieldData<?> d) {
|
||||
Objects.requireNonNull(d);
|
||||
|
||||
FieldDataValueSource(IndexFieldData<?> d) {
|
||||
fieldData = d;
|
||||
}
|
||||
|
||||
|
|
@ -49,8 +51,13 @@ class FieldDataValueSource extends ValueSource {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
return fieldData.equals(other);
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
FieldDataValueSource that = (FieldDataValueSource) o;
|
||||
|
||||
return fieldData.equals(that.fieldData);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -61,6 +61,15 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
|
|||
}
|
||||
|
||||
public void testBasic() throws Exception {
|
||||
createIndex("test");
|
||||
ensureGreen("test");
|
||||
client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get();
|
||||
SearchResponse rsp = buildRequest("doc['foo'] + 1").get();
|
||||
assertEquals(1, rsp.getHits().getTotalHits());
|
||||
assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue());
|
||||
}
|
||||
|
||||
public void testBasicUsingDotValue() throws Exception {
|
||||
createIndex("test");
|
||||
ensureGreen("test");
|
||||
client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get();
|
||||
|
|
@ -89,13 +98,56 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
|
|||
assertEquals("2", hits.getAt(2).getId());
|
||||
}
|
||||
|
||||
public void testDateMethods() throws Exception {
|
||||
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
|
||||
ensureGreen("test");
|
||||
indexRandom(true,
|
||||
client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
|
||||
client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
|
||||
SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get();
|
||||
assertEquals(2, rsp.getHits().getTotalHits());
|
||||
SearchHits hits = rsp.getHits();
|
||||
assertEquals(5.0, hits.getAt(0).field("foo").getValue());
|
||||
assertEquals(-11.0, hits.getAt(1).field("foo").getValue());
|
||||
rsp = buildRequest("doc['date0'].getHourOfDay() + doc['date1'].getDayOfMonth()").get();
|
||||
assertEquals(2, rsp.getHits().getTotalHits());
|
||||
hits = rsp.getHits();
|
||||
assertEquals(5.0, hits.getAt(0).field("foo").getValue());
|
||||
assertEquals(24.0, hits.getAt(1).field("foo").getValue());
|
||||
rsp = buildRequest("doc['date1'].getMonth() + 1").get();
|
||||
assertEquals(2, rsp.getHits().getTotalHits());
|
||||
hits = rsp.getHits();
|
||||
assertEquals(9.0, hits.getAt(0).field("foo").getValue());
|
||||
assertEquals(10.0, hits.getAt(1).field("foo").getValue());
|
||||
rsp = buildRequest("doc['date1'].getYear()").get();
|
||||
assertEquals(2, rsp.getHits().getTotalHits());
|
||||
hits = rsp.getHits();
|
||||
assertEquals(1985.0, hits.getAt(0).field("foo").getValue());
|
||||
assertEquals(1983.0, hits.getAt(1).field("foo").getValue());
|
||||
}
|
||||
|
||||
public void testInvalidDateMethodCall() throws Exception {
|
||||
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double", "type=double"));
|
||||
ensureGreen("test");
|
||||
indexRandom(true, client().prepareIndex("test", "doc", "1").setSource("double", "178000000.0"));
|
||||
try {
|
||||
buildRequest("doc['double'].getYear()").get();
|
||||
fail();
|
||||
} catch (SearchPhaseExecutionException e) {
|
||||
assertThat(e.toString() + "should have contained IllegalArgumentException",
|
||||
e.toString().contains("IllegalArgumentException"), equalTo(true));
|
||||
assertThat(e.toString() + "should have contained can only be used with a date field type",
|
||||
e.toString().contains("can only be used with a date field type"), equalTo(true));
|
||||
}
|
||||
}
|
||||
|
||||
public void testSparseField() throws Exception {
|
||||
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long"));
|
||||
ensureGreen("test");
|
||||
indexRandom(true,
|
||||
client().prepareIndex("test", "doc", "1").setSource("x", 4),
|
||||
client().prepareIndex("test", "doc", "2").setSource("y", 2));
|
||||
SearchResponse rsp = buildRequest("doc['x'].value + 1").get();
|
||||
SearchResponse rsp = buildRequest("doc['x'] + 1").get();
|
||||
ElasticsearchAssertions.assertSearchResponse(rsp);
|
||||
SearchHits hits = rsp.getHits();
|
||||
assertEquals(2, rsp.getHits().getTotalHits());
|
||||
|
|
@ -108,7 +160,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
|
|||
ensureGreen("test");
|
||||
client().prepareIndex("test", "doc", "1").setSource("x", 4).setRefresh(true).get();
|
||||
try {
|
||||
buildRequest("doc['bogus'].value").get();
|
||||
buildRequest("doc['bogus']").get();
|
||||
fail("Expected missing field to cause failure");
|
||||
} catch (SearchPhaseExecutionException e) {
|
||||
assertThat(e.toString() + "should have contained ExpressionScriptCompilationException",
|
||||
|
|
@ -126,7 +178,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
|
|||
client().prepareIndex("test", "doc", "2").setSource("x", 3),
|
||||
client().prepareIndex("test", "doc", "3").setSource("x", 5));
|
||||
// a = int, b = double, c = long
|
||||
String script = "doc['x'].value * a + b + ((c + doc['x'].value) > 5000000009 ? 1 : 0)";
|
||||
String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
|
||||
SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get();
|
||||
SearchHits hits = rsp.getHits();
|
||||
assertEquals(3, hits.getTotalHits());
|
||||
|
|
@ -164,7 +216,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
|
|||
public void testNonNumericField() {
|
||||
client().prepareIndex("test", "doc", "1").setSource("text", "this is not a number").setRefresh(true).get();
|
||||
try {
|
||||
buildRequest("doc['text'].value").get();
|
||||
buildRequest("doc['text']").get();
|
||||
fail("Expected text field to cause execution failure");
|
||||
} catch (SearchPhaseExecutionException e) {
|
||||
assertThat(e.toString() + "should have contained ExpressionScriptCompilationException",
|
||||
|
|
@ -208,8 +260,8 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
|
|||
} catch (SearchPhaseExecutionException e) {
|
||||
assertThat(e.toString() + "should have contained ExpressionScriptCompilationException",
|
||||
e.toString().contains("ExpressionScriptCompilationException"), equalTo(true));
|
||||
assertThat(e.toString() + "should have contained field member error",
|
||||
e.toString().contains("Invalid member for field"), equalTo(true));
|
||||
assertThat(e.toString() + "should have contained member variable [value] or member methods may be accessed",
|
||||
e.toString().contains("member variable [value] or member methods may be accessed"), equalTo(true));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue