Make dates be ReadableDateTimes in scripts (#22948)

Instead of longs. If you want millis since epoch you can call doc.date_field.value.millis.

Relates to #22875

core/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java

@@ -30,7 +30,7 @@ public interface AtomicFieldData extends Accountable, Releasable {
     /**
      * Returns a "scripting" based values.
      */
-    ScriptDocValues getScriptValues();
+    ScriptDocValues<?> getScriptValues();
 
     /**
      * Return a String representation of the values.

core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java

@@ -27,6 +27,7 @@ public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericField
         SHORT(false),
         INT(false),
         LONG(false),
+        DATE(false),
         HALF_FLOAT(true),
         FLOAT(true),
         DOUBLE(true);

core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java

@@ -25,6 +25,8 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
@@ -41,7 +43,6 @@ import java.util.function.UnaryOperator;
  * and a <code>getValues</code> that return the relevant type that then can be used in scripts.
  */
 public abstract class ScriptDocValues<T> extends AbstractList<T> {
-
     /**
      * Set the current doc ID.
      */
@@ -127,6 +128,7 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
     }
 
     public static final class Longs extends ScriptDocValues<Long> {
+        protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Longs.class));
 
         private final SortedNumericDocValues values;
         private Dates dates;
@@ -155,7 +157,9 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
             return values.valueAt(0);
         }
 
+        @Deprecated
         public ReadableDateTime getDate() {
+            deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates.");
             if (dates == null) {
                 dates = new Dates(values);
                 dates.refreshArray();
@@ -163,7 +167,9 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
             return dates.getValue();
         }
 
+        @Deprecated
         public List<ReadableDateTime> getDates() {
+            deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates.");
             if (dates == null) {
                 dates = new Dates(values);
                 dates.refreshArray();
@@ -183,6 +189,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
     }
 
     public static final class Dates extends ScriptDocValues<ReadableDateTime> {
+        protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class));
+
         private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);
 
         private final SortedNumericDocValues values;
@@ -206,6 +214,24 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
             return get(0);
         }
 
+        /**
+         * Fetch the first value. Added for backwards compatibility with 5.x when date fields were {@link Longs}.
+         */
+        @Deprecated
+        public ReadableDateTime getDate() {
+            deprecationLogger.deprecated("getDate is no longer necisary on date fields as the value is now a date.");
+            return getValue();
+        }
+
+        /**
+         * Fetch all the values. Added for backwards compatibility with 5.x when date fields were {@link Longs}.
+         */
+        @Deprecated
+        public List<ReadableDateTime> getDates() {
+            deprecationLogger.deprecated("getDates is no longer necisary on date fields as the values are now dates.");
+            return this;
+        }
+
         @Override
         public ReadableDateTime get(int index) {
             if (index >= values.count()) {

core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java

@@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata.plain;
 
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
@@ -31,12 +32,14 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 abstract class AtomicLongFieldData implements AtomicNumericFieldData {
 
     private final long ramBytesUsed;
-    /** True if this numeric data is for a boolean field, and so only has values 0 and 1. */
+    /**
-    private final boolean isBoolean;
+     * Type of this field. Used to expose appropriate types in {@link #getScriptValues()}.
+     */
+    private final NumericType numericType;
 
-    AtomicLongFieldData(long ramBytesUsed, boolean isBoolean) {
+    AtomicLongFieldData(long ramBytesUsed, NumericType numericType) {
         this.ramBytesUsed = ramBytesUsed;
-        this.isBoolean = isBoolean;
+        this.numericType = numericType;
     }
 
     @Override
@@ -45,10 +48,13 @@ abstract class AtomicLongFieldData implements AtomicNumericFieldData {
     }
 
     @Override
-    public final ScriptDocValues getScriptValues() {
+    public final ScriptDocValues<?> getScriptValues() {
-        if (isBoolean) {
+        switch (numericType) {
+        case DATE:
+            return new ScriptDocValues.Dates(getLongValues());
+        case BOOLEAN:
             return new ScriptDocValues.Booleans(getLongValues());
-        } else {
+        default:
             return new ScriptDocValues.Longs(getLongValues());
         }
     }

core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java

@@ -96,7 +96,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple
             case DOUBLE:
                 return new SortedNumericDoubleFieldData(reader, field);
             default:
-                return new SortedNumericLongFieldData(reader, field, numericType == NumericType.BOOLEAN);
+                return new SortedNumericLongFieldData(reader, field, numericType);
         }
     }
 
@@ -117,8 +117,8 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple
         final LeafReader reader;
         final String field;
 
-        SortedNumericLongFieldData(LeafReader reader, String field, boolean isBoolean) {
+        SortedNumericLongFieldData(LeafReader reader, String field, NumericType numericType) {
-            super(0L, isBoolean);
+            super(0L, numericType);
             this.reader = reader;
             this.field = field;
         }

core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -376,7 +376,7 @@ public class DateFieldMapper extends FieldMapper {
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
             failIfNoDocValues();
-            return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.DATE);
         }
 
         @Override

core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java

@@ -23,10 +23,10 @@ import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.joda.time.ReadableInstant;
 
 import java.lang.reflect.Array;
 import java.util.Collection;
-import java.util.Iterator;
 
 /**
  * {@link SortingNumericDoubleValues} implementation which is based on a script
@@ -47,36 +47,49 @@ public class ScriptDoubleValues extends SortingNumericDoubleValues implements Sc
 
         if (value == null) {
             resize(0);
-        }
+        } else if (value instanceof Number) {
-
-        else if (value instanceof Number) {
             resize(1);
             values[0] = ((Number) value).doubleValue();
-        }
+        } else if (value instanceof ReadableInstant) {
-
+            resize(1);
-        else if (value.getClass().isArray()) {
+            values[0] = ((ReadableInstant) value).getMillis();
+        } else if (value.getClass().isArray()) {
             resize(Array.getLength(value));
             for (int i = 0; i < count(); ++i) {
-                values[i] = ((Number) Array.get(value, i)).doubleValue();
+                values[i] = toDoubleValue(Array.get(value, i));
             }
-        }
+        } else if (value instanceof Collection) {
-
-        else if (value instanceof Collection) {
             resize(((Collection<?>) value).size());
             int i = 0;
-            for (Iterator<?> it = ((Collection<?>) value).iterator(); it.hasNext(); ++i) {
+            for (Object v : (Collection<?>) value) {
-                values[i] = ((Number) it.next()).doubleValue();
+                values[i++] = toDoubleValue(v);
             }
             assert i == count();
-        }
+        } else {
-
+            resize(1);
-        else {
+            values[0] = toDoubleValue(value);
-            throw new AggregationExecutionException("Unsupported script value [" + value + "]");
         }
 
         sort();
     }
 
+    private static double toDoubleValue(Object o) {
+        if (o instanceof Number) {
+            return ((Number) o).doubleValue();
+        } else if (o instanceof ReadableInstant) {
+            // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
+            return ((ReadableInstant) o).getMillis();
+        } else if (o instanceof Boolean) {
+            // We do expose boolean fields as boolean in scripts, however aggregations still expect
+            // that scripts return the same internal representation as regular fields, so boolean
+            // values in scripts need to be converted to a number, and the value formatter will
+            // make sure of using true/false in the key_as_string field
+            return ((Boolean) o).booleanValue() ? 1.0 : 0.0;
+        } else {
+            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number, date, or boolean");
+        }
+    }
+
     @Override
     public void setScorer(Scorer scorer) {
         script.setScorer(scorer);

core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java

@@ -24,6 +24,7 @@ import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.index.fielddata.SortingNumericDocValues;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.joda.time.ReadableInstant;
 
 import java.lang.reflect.Array;
 import java.util.Collection;
@@ -77,6 +78,9 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA
     private static long toLongValue(Object o) {
         if (o instanceof Number) {
             return ((Number) o).longValue();
+        } else if (o instanceof ReadableInstant) {
+            // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
+            return ((ReadableInstant) o).getMillis();
         } else if (o instanceof Boolean) {
             // We do expose boolean fields as boolean in scripts, however aggregations still expect
             // that scripts return the same internal representation as regular fields, so boolean
@@ -84,7 +88,7 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA
             // make sure of using true/false in the key_as_string field
             return ((Boolean) o).booleanValue() ? 1L : 0L;
         } else {
-            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number");
+            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number, date, or boolean");
         }
     }
 

core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java

@@ -62,10 +62,12 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase {
             }
             MappedFieldType fieldType = context.mapperService().fullName(field);
             if (fieldType != null) {
+                /* Because this is called once per document we end up creating a new ScriptDocValues for every document which is important
+                 * because the values inside ScriptDocValues might be reused for different documents (Dates do this). */
                 AtomicFieldData data = context.fieldData().getForField(fieldType).load(hitContext.readerContext());
-                ScriptDocValues values = data.getScriptValues();
+                ScriptDocValues<?> values = data.getScriptValues();
                 values.setNextDocId(hitContext.docId());
-                hitField.values().addAll(values.getValues());
+                hitField.values().addAll(values);
             }
         }
     }

core/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java

@@ -39,6 +39,8 @@ public final class ScriptFieldsFetchSubPhase implements FetchSubPhase {
             return;
         }
         for (ScriptFieldsContext.ScriptField scriptField : context.scriptFields().fields()) {
+            /* Because this is called once per document we end up creating new ScriptDocValues for every document which is important because
+             * the values inside ScriptDocValues might be reused for different documents (Dates do this). */
             LeafSearchScript leafScript;
             try {
                 leafScript = scriptField.script().getLeafSearchScript(hitContext.readerContext());

core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java

@@ -80,6 +80,10 @@ public class ScriptDocValuesLongsTests extends ESTestCase {
             Exception e = expectThrows(UnsupportedOperationException.class, () -> longs.getDates().add(new DateTime()));
             assertEquals("doc values are unmodifiable", e.getMessage());
         }
+
+        assertWarnings(
+                "getDate on numeric fields is deprecated. Use a date field to get dates.",
+                "getDates on numeric fields is deprecated. Use a date field to get dates.");
     }
 
     private Longs wrap(long[][] values) {

core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java

@@ -84,7 +84,7 @@ public class DateScriptMocks {
 
         @Override
         public ExecutableScript newScript(Map<String, Object> params) {
-            return new PlusOneMonthScript((String) params.get("fieldname"));
+            return new PlusOneMonthScript();
         }
 
         @Override
@@ -104,14 +104,9 @@ public class DateScriptMocks {
     public static class PlusOneMonthScript extends AbstractSearchScript {
 
         public static final String NAME = "date_plus_1_month";
-        private String fieldname;
 
         private Map<String, Object> vars = new HashMap<>();
 
-        public PlusOneMonthScript(String fieldname) {
-            this.fieldname  = fieldname;
-        }
-
         @Override
         public void setNextVar(String name, Object value) {
             vars.put(name, value);

core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java

@@ -45,6 +45,7 @@ import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.ReadableDateTime;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -105,8 +106,8 @@ public class SearchFieldsIT extends ESIntegTestCase {
 
             scripts.put("doc['date'].date.millis", vars -> {
                 Map<?, ?> doc = (Map) vars.get("doc");
-                ScriptDocValues.Longs date = (ScriptDocValues.Longs) doc.get("date");
+                ScriptDocValues.Dates dates = (ScriptDocValues.Dates) doc.get("date");
-                return date.getDate().getMillis();
+                return dates.getValue().getMillis();
             });
 
             scripts.put("_fields['num1'].value", vars -> fieldsScript(vars, "num1"));
@@ -777,6 +778,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
 
         client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet();
 
+        ReadableDateTime date = new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC);
         client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
                 .field("text_field", "foo")
                 .field("keyword_field", "foo")
@@ -786,7 +788,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
                 .field("long_field", 4L)
                 .field("float_field", 5.0f)
                 .field("double_field", 6.0d)
-                .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)))
+                .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
                 .field("boolean_field", true)
                 .field("ip_field", "::1")
                 .endObject()).execute().actionGet();
@@ -820,7 +822,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).fields().get("long_field").value(), equalTo((Object) 4L));
         assertThat(searchResponse.getHits().getAt(0).fields().get("float_field").value(), equalTo((Object) 5.0));
         assertThat(searchResponse.getHits().getAt(0).fields().get("double_field").value(), equalTo((Object) 6.0d));
-        assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) 1332374400000L));
+        assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo(date));
         assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) true));
         assertThat(searchResponse.getHits().getAt(0).fields().get("text_field").value(), equalTo("foo"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("keyword_field").value(), equalTo("foo"));

docs/reference/migration/migrate_6_0/scripting.asciidoc

@@ -5,3 +5,10 @@
 
 The groovy scripting language was deprecated in elasticsearch 5.0 and is now removed.
 Use painless instead.
+
+==== Date fields now return dates
+
+`doc.some_date_field.value` now returns `ReadableDateTime`s instead of
+milliseconds since epoch as a `long`. The same is true for
+`doc.some_date_field[some_number]`. Use `doc.some_date_field.value.millis` to
+fetch the milliseconds since epoch if you need it.

docs/reference/modules/scripting/painless.asciidoc

@@ -196,10 +196,17 @@ POST hockey/player/1/_update
 
 [float]
 [[modules-scripting-painless-dates]]
-=== Regular expressions
+=== Dates
 
-Dates are a little different to work with than regular values. Here is an
+Date fields are exposed as
-example returning the year of every player's birth:
+<<painless-api-reference-org-joda-time-ReadableDateTime, `ReadableDateTime`>>s
+so they support methods like
+<<painless-api-reference-org-joda-time-ReadableDateTime-getYear-0, `getYear`>>,
+and
+<<painless-api-reference-org-joda-time-ReadableDateTime-getDayOfWeek-0, `getDayOfWeek`>>.
+To get milliseconds since epoch call
+<<painless-api-reference-org-joda-time-ReadableInstant-getMillis-0, `getMillis`>>.
+For example, the following returns every hockey player's birth year:
 
 [source,js]
 ----------------------------------------------------------------
@@ -208,7 +215,7 @@ GET hockey/_search
   "script_fields": {
     "birth_year": {
       "script": {
-        "inline": "doc.born.date.year"
+        "inline": "doc.born.value.year"
       }
     }
   }
@@ -216,18 +223,6 @@ GET hockey/_search
 ----------------------------------------------------------------
 // CONSOLE
 
-The key here is that instead of indexing directly into `doc.born` like you would
-a normal field you have to call `doc.born.date` to get a
-<<painless-api-reference-org-joda-time-ReadableDateTime, `ReadableDateTime`>>.
-From there you can call methods like
-<<painless-api-reference-org-joda-time-ReadableDateTime-getYear-0, `getYear`>>,
-and <<painless-api-reference-org-joda-time-ReadableDateTime-getDayOfWeek-0, `getDayOfWeek`>>.
-In the example above `year` is a shortcut to `getYear()`.
-
-If the date field is a list then `date` will always return the first date. To
-access all the dates use `dates` instead of `date`.
-
-
 [float]
 [[modules-scripting-painless-regex]]
 === Regular expressions

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt

@@ -85,6 +85,14 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsear
   List getDates()
 }
 
+class org.elasticsearch.index.fielddata.ScriptDocValues.Dates -> org.elasticsearch.index.fielddata.ScriptDocValues$Dates extends List,Collection,Iterable,Object {
+  org.joda.time.ReadableDateTime get(int)
+  org.joda.time.ReadableDateTime getValue()
+  List getValues()
+  org.joda.time.ReadableDateTime getDate()
+  List getDates()
+}
+
 class org.elasticsearch.index.fielddata.ScriptDocValues.Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
   Double get(int)
   double getValue()

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml

@@ -109,7 +109,7 @@ setup:
                 script_fields:
                     bar:
                         script:
-                            inline: "doc.date.date.dayOfWeek"
+                            inline: "doc.date.value.dayOfWeek"
 
     - match: { hits.hits.0.fields.bar.0: 7}
 
@@ -123,7 +123,7 @@ setup:
                         script:
                             inline: >
                               StringBuilder b = new StringBuilder();
-                              for (def date : doc.dates.dates) {
+                              for (def date : doc.dates) {
                                 b.append(" ").append(date.getDayOfWeek());
                               }
                               return b.toString().trim()

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml

@@ -113,8 +113,8 @@ setup:
                 script_fields:
                     field:
                         script:
-                            inline: "doc['date'].get(0)"
+                            inline: "doc.date.get(0)"
-    - match: { hits.hits.0.fields.field.0: 1483272672000 }
+    - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
 
     - do:
         search:
@@ -122,8 +122,8 @@ setup:
                 script_fields:
                     field:
                         script:
-                            inline: "doc['date'].value"
+                            inline: "doc.date.value"
-    - match: { hits.hits.0.fields.field.0: 1483272672000 }
+    - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
 
 ---
 "geo_point":