Scripting: Add Field Methods

Added infrastructure to allow basic member methods in the expressions
language to be called.  The methods must have a signature with no arguments.  Also
added the following member methods for date fields (and it should be easy to add more)
* getYear
* getMonth
* getDayOfMonth
* getHourOfDay
* getMinutes
* getSeconds

Allow fields to be accessed without using the member variable [value].
(Note that both ways can be used to access fields for back-compat.)

closes #10890

docs/reference/modules/scripting.asciidoc

@@ -389,9 +389,23 @@ for details on what operators and functions are available.
 Variables in `expression` scripts are available to access:
 
 * Single valued document fields, e.g. `doc['myfield'].value`
+* Single valued document fields can also be accessed without `.value` e.g. `doc['myfield']`
 * Parameters passed into the script, e.g. `mymodifier`
 * The current document's score, `_score` (only available when used in a `script_score`)
 
+Variables in `expression` scripts that are of type `date` may use the following member methods:
+
+* getYear()
+* getMonth()
+* getDayOfMonth()
+* getHourOfDay()
+* getMinutes()
+* getSeconds()
+
+The following example shows the difference in years between the `date` fields date0 and date1:
+
+`doc['date1'].getYear() - doc['date0'].getYear()`
+
 There are a few limitations relative to other script languages:
 
 * Only numeric fields may be accessed

src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
+
+import org.apache.lucene.queries.function.ValueSource;
+import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+
+class DateMethodFunctionValues extends FieldDataFunctionValues {
+    private final int calendarType;
+    private final Calendar calendar;
+
+    DateMethodFunctionValues(ValueSource parent, AtomicNumericFieldData data, int calendarType) {
+        super(parent, data);
+
+        this.calendarType = calendarType;
+        calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
+    }
+
+    @Override
+    public double doubleVal(int docId) {
+        long millis = (long)dataAccessor.get(docId);
+        calendar.setTimeInMillis(millis);
+        return calendar.get(calendarType);
+    }
+}

src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+
+import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+
+class DateMethodValueSource extends FieldDataValueSource {
+
+    protected final String methodName;
+    protected final int calendarType;
+
+    DateMethodValueSource(IndexFieldData<?> indexFieldData, String methodName, int calendarType) {
+        super(indexFieldData);
+
+        Objects.requireNonNull(methodName);
+
+        this.methodName = methodName;
+        this.calendarType = calendarType;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+
+        DateMethodValueSource that = (DateMethodValueSource) o;
+
+        if (calendarType != that.calendarType) return false;
+        return methodName.equals(that.methodName);
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + methodName.hashCode();
+        result = 31 * result + calendarType;
+        return result;
+    }
+
+    @Override
+    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+        AtomicFieldData leafData = fieldData.load(leaf);
+        assert(leafData instanceof AtomicNumericFieldData);
+
+        return new DateMethodFunctionValues(this, (AtomicNumericFieldData)leafData, calendarType);
+    }
+
+    @Override
+    public String description() {
+        return methodName + ": field(" + fieldData.getFieldNames().toString() + ")";
+    }
+}

src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java

@@ -23,6 +23,7 @@ import org.apache.lucene.expressions.Expression;
 import org.apache.lucene.expressions.SimpleBindings;
 import org.apache.lucene.expressions.js.JavascriptCompiler;
 import org.apache.lucene.expressions.js.VariableContext;
+import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.Nullable;
@@ -32,6 +33,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.core.DateFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
@@ -40,6 +42,7 @@ import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.text.ParseException;
+import java.util.Calendar;
 import java.util.Map;
 
 /**
@@ -50,6 +53,13 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
 
     public static final String NAME = "expression";
 
+    protected static final String GET_YEAR_METHOD         = "getYear";
+    protected static final String GET_MONTH_METHOD        = "getMonth";
+    protected static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth";
+    protected static final String GET_HOUR_OF_DAY_METHOD  = "getHourOfDay";
+    protected static final String GET_MINUTES_METHOD      = "getMinutes";
+    protected static final String GET_SECONDS_METHOD      = "getSeconds";
+
     @Inject
     public ExpressionScriptEngineService(Settings settings) {
         super(settings);
@@ -112,19 +122,30 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                 }
 
             } else {
+                String fieldname = null;
+                String methodname = null;
                 VariableContext[] parts = VariableContext.parse(variable);
                 if (parts[0].text.equals("doc") == false) {
                     throw new ExpressionScriptCompilationException("Unknown variable [" + parts[0].text + "] in expression");
                 }
                 if (parts.length < 2 || parts[1].type != VariableContext.Type.STR_INDEX) {
-                    throw new ExpressionScriptCompilationException("Variable 'doc' in expression must be used with a specific field like: doc['myfield'].value");
+                    throw new ExpressionScriptCompilationException("Variable 'doc' in expression must be used with a specific field like: doc['myfield']");
+                } else {
+                    fieldname = parts[1].text;
                 }
-                if (parts.length < 3 || parts[2].type != VariableContext.Type.MEMBER || parts[2].text.equals("value") == false) {
+                if (parts.length == 3) {
-                    throw new ExpressionScriptCompilationException("Invalid member for field data in expression.  Only '.value' is currently supported.");
+                    if (parts[2].type == VariableContext.Type.METHOD) {
+                        methodname = parts[2].text;
+                    } else if (parts[2].type != VariableContext.Type.MEMBER || !"value".equals(parts[2].text)) {
+                        throw new ExpressionScriptCompilationException("Only the member variable [value] or member methods may be accessed on a field when not accessing the field directly");
+                    }
+                }
+                if (parts.length > 3) {
+                    throw new ExpressionScriptCompilationException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
                 }
-                String fieldname = parts[1].text;
 
                 FieldMapper<?> field = mapper.smartNameFieldMapper(fieldname);
+
                 if (field == null) {
                     throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression does not exist in mappings");
                 }
@@ -132,14 +153,46 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     // TODO: more context (which expression?)
                     throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression must be numeric");
                 }
+
                 IndexFieldData<?> fieldData = lookup.doc().fieldDataService().getForField((NumberFieldMapper)field);
-                bindings.add(variable, new FieldDataValueSource(fieldData));
+                if (methodname == null) {
+                    bindings.add(variable, new FieldDataValueSource(fieldData));
+                } else {
+                    bindings.add(variable, getMethodValueSource(field, fieldData, fieldname, methodname));
+                }
             }
         }
 
         return new ExpressionScript((Expression)compiledScript, bindings, specialValue);
     }
 
+    protected ValueSource getMethodValueSource(FieldMapper<?> field, IndexFieldData<?> fieldData, String fieldName, String methodName) {
+        switch (methodName) {
+            case GET_YEAR_METHOD:
+                return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.YEAR);
+            case GET_MONTH_METHOD:
+                return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.MONTH);
+            case GET_DAY_OF_MONTH_METHOD:
+                return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.DAY_OF_MONTH);
+            case GET_HOUR_OF_DAY_METHOD:
+                return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.HOUR_OF_DAY);
+            case GET_MINUTES_METHOD:
+                return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.MINUTE);
+            case GET_SECONDS_METHOD:
+                return getDateMethodValueSource(field, fieldData, fieldName, methodName, Calendar.SECOND);
+            default:
+                throw new IllegalArgumentException("Member method [" + methodName + "] does not exist.");
+        }
+    }
+
+    protected ValueSource getDateMethodValueSource(FieldMapper<?> field, IndexFieldData<?> fieldData, String fieldName, String methodName, int calendarType) {
+        if (!(field instanceof DateFieldMapper)) {
+            throw new IllegalArgumentException("Member method [" + methodName + "] can only be used with a date field type, not the field [" + fieldName + "].");
+        }
+
+        return new DateMethodValueSource(fieldData, methodName, calendarType);
+    }
+
     @Override
     public ExecutableScript executable(Object compiledScript, @Nullable Map<String, Object> vars) {
         throw new UnsupportedOperationException("Cannot use expressions for updates");

src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.script.expression;
 
-
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
@@ -29,15 +28,18 @@ import org.elasticsearch.index.fielddata.IndexFieldData;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * A {@link ValueSource} wrapper for field data.
  */
 class FieldDataValueSource extends ValueSource {
 
-    IndexFieldData<?> fieldData;
+    protected IndexFieldData<?> fieldData;
+
+    protected FieldDataValueSource(IndexFieldData<?> d) {
+        Objects.requireNonNull(d);
 
-    FieldDataValueSource(IndexFieldData<?> d) {
         fieldData = d;
     }
 
@@ -49,8 +51,13 @@ class FieldDataValueSource extends ValueSource {
     }
 
     @Override
-    public boolean equals(Object other) {
+    public boolean equals(Object o) {
-        return fieldData.equals(other);
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FieldDataValueSource that = (FieldDataValueSource) o;
+
+        return fieldData.equals(that.fieldData);
     }
 
     @Override

src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java

@@ -61,6 +61,15 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
     }
 
     public void testBasic() throws Exception {
+        createIndex("test");
+        ensureGreen("test");
+        client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get();
+        SearchResponse rsp = buildRequest("doc['foo'] + 1").get();
+        assertEquals(1, rsp.getHits().getTotalHits());
+        assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue());
+    }
+
+    public void testBasicUsingDotValue() throws Exception {
         createIndex("test");
         ensureGreen("test");
         client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get();
@@ -89,13 +98,56 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
         assertEquals("2", hits.getAt(2).getId());
     }
 
+    public void testDateMethods() throws Exception {
+        ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
+        ensureGreen("test");
+        indexRandom(true,
+                client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
+                client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
+        SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        SearchHits hits = rsp.getHits();
+        assertEquals(5.0, hits.getAt(0).field("foo").getValue());
+        assertEquals(-11.0, hits.getAt(1).field("foo").getValue());
+        rsp = buildRequest("doc['date0'].getHourOfDay() + doc['date1'].getDayOfMonth()").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        hits = rsp.getHits();
+        assertEquals(5.0, hits.getAt(0).field("foo").getValue());
+        assertEquals(24.0, hits.getAt(1).field("foo").getValue());
+        rsp = buildRequest("doc['date1'].getMonth() + 1").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        hits = rsp.getHits();
+        assertEquals(9.0, hits.getAt(0).field("foo").getValue());
+        assertEquals(10.0, hits.getAt(1).field("foo").getValue());
+        rsp = buildRequest("doc['date1'].getYear()").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        hits = rsp.getHits();
+        assertEquals(1985.0, hits.getAt(0).field("foo").getValue());
+        assertEquals(1983.0, hits.getAt(1).field("foo").getValue());
+    }
+
+    public void testInvalidDateMethodCall() throws Exception {
+        ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double", "type=double"));
+        ensureGreen("test");
+        indexRandom(true, client().prepareIndex("test", "doc", "1").setSource("double", "178000000.0"));
+        try {
+            buildRequest("doc['double'].getYear()").get();
+            fail();
+        } catch (SearchPhaseExecutionException e) {
+            assertThat(e.toString() + "should have contained IllegalArgumentException",
+                    e.toString().contains("IllegalArgumentException"), equalTo(true));
+            assertThat(e.toString() + "should have contained can only be used with a date field type",
+                    e.toString().contains("can only be used with a date field type"), equalTo(true));
+        }
+    }
+
     public void testSparseField() throws Exception {
         ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long"));
         ensureGreen("test");
         indexRandom(true,
                 client().prepareIndex("test", "doc", "1").setSource("x", 4),
                 client().prepareIndex("test", "doc", "2").setSource("y", 2));
-        SearchResponse rsp = buildRequest("doc['x'].value + 1").get();
+        SearchResponse rsp = buildRequest("doc['x'] + 1").get();
         ElasticsearchAssertions.assertSearchResponse(rsp);
         SearchHits hits = rsp.getHits();
         assertEquals(2, rsp.getHits().getTotalHits());
@@ -108,7 +160,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
         ensureGreen("test");
         client().prepareIndex("test", "doc", "1").setSource("x", 4).setRefresh(true).get();
         try {
-            buildRequest("doc['bogus'].value").get();
+            buildRequest("doc['bogus']").get();
             fail("Expected missing field to cause failure");
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should have contained ExpressionScriptCompilationException",
@@ -126,7 +178,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
                     client().prepareIndex("test", "doc", "2").setSource("x", 3),
                     client().prepareIndex("test", "doc", "3").setSource("x", 5));
         // a = int, b = double, c = long
-        String script = "doc['x'].value * a + b + ((c + doc['x'].value) > 5000000009 ? 1 : 0)";
+        String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
         SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get();
         SearchHits hits = rsp.getHits();
         assertEquals(3, hits.getTotalHits());
@@ -164,7 +216,7 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
     public void testNonNumericField() {
         client().prepareIndex("test", "doc", "1").setSource("text", "this is not a number").setRefresh(true).get();
         try {
-            buildRequest("doc['text'].value").get();
+            buildRequest("doc['text']").get();
             fail("Expected text field to cause execution failure");
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should have contained ExpressionScriptCompilationException",
@@ -208,8 +260,8 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest {
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should have contained ExpressionScriptCompilationException",
                     e.toString().contains("ExpressionScriptCompilationException"), equalTo(true));
-            assertThat(e.toString() + "should have contained field member error",
+            assertThat(e.toString() + "should have contained member variable [value] or member methods may be accessed",
-                    e.toString().contains("Invalid member for field"), equalTo(true));
+                    e.toString().contains("member variable [value] or member methods may be accessed"), equalTo(true));
         }
     }