Make dates be ReadableDateTimes in scripts (#22948)

Instead of longs. If you want millis since epoch you can call doc.date_field.value.millis.

Relates to #22875
This commit is contained in:
Nik Everett 2017-02-06 16:44:56 -05:00 committed by GitHub
parent bc884c1e7b
commit 0d6e622242
18 changed files with 130 additions and 65 deletions

View File

@ -30,7 +30,7 @@ public interface AtomicFieldData extends Accountable, Releasable {
/**
* Returns a "scripting" based values.
*/
ScriptDocValues getScriptValues();
ScriptDocValues<?> getScriptValues();
/**
* Return a String representation of the values.

View File

@ -27,6 +27,7 @@ public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericField
SHORT(false),
INT(false),
LONG(false),
DATE(false),
HALF_FLOAT(true),
FLOAT(true),
DOUBLE(true);

View File

@ -25,6 +25,8 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.geo.GeoHashUtils;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.MutableDateTime;
@ -41,7 +43,6 @@ import java.util.function.UnaryOperator;
* and a <code>getValues</code> that return the relevant type that then can be used in scripts.
*/
public abstract class ScriptDocValues<T> extends AbstractList<T> {
/**
* Set the current doc ID.
*/
@ -127,6 +128,7 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
}
public static final class Longs extends ScriptDocValues<Long> {
protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Longs.class));
private final SortedNumericDocValues values;
private Dates dates;
@ -155,7 +157,9 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
return values.valueAt(0);
}
@Deprecated
public ReadableDateTime getDate() {
deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates.");
if (dates == null) {
dates = new Dates(values);
dates.refreshArray();
@ -163,7 +167,9 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
return dates.getValue();
}
@Deprecated
public List<ReadableDateTime> getDates() {
deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates.");
if (dates == null) {
dates = new Dates(values);
dates.refreshArray();
@ -183,6 +189,8 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
}
public static final class Dates extends ScriptDocValues<ReadableDateTime> {
protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class));
private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);
private final SortedNumericDocValues values;
@ -206,6 +214,24 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
return get(0);
}
/**
* Fetch the first value. Added for backwards compatibility with 5.x when date fields were {@link Longs}.
*/
@Deprecated
public ReadableDateTime getDate() {
deprecationLogger.deprecated("getDate is no longer necisary on date fields as the value is now a date.");
return getValue();
}
/**
* Fetch all the values. Added for backwards compatibility with 5.x when date fields were {@link Longs}.
*/
@Deprecated
public List<ReadableDateTime> getDates() {
deprecationLogger.deprecated("getDates is no longer necisary on date fields as the values are now dates.");
return this;
}
@Override
public ReadableDateTime get(int index) {
if (index >= values.count()) {

View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata.plain;
import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
@ -31,12 +32,14 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
abstract class AtomicLongFieldData implements AtomicNumericFieldData {
private final long ramBytesUsed;
/** True if this numeric data is for a boolean field, and so only has values 0 and 1. */
private final boolean isBoolean;
/**
* Type of this field. Used to expose appropriate types in {@link #getScriptValues()}.
*/
private final NumericType numericType;
AtomicLongFieldData(long ramBytesUsed, boolean isBoolean) {
AtomicLongFieldData(long ramBytesUsed, NumericType numericType) {
this.ramBytesUsed = ramBytesUsed;
this.isBoolean = isBoolean;
this.numericType = numericType;
}
@Override
@ -45,10 +48,13 @@ abstract class AtomicLongFieldData implements AtomicNumericFieldData {
}
@Override
public final ScriptDocValues getScriptValues() {
if (isBoolean) {
public final ScriptDocValues<?> getScriptValues() {
switch (numericType) {
case DATE:
return new ScriptDocValues.Dates(getLongValues());
case BOOLEAN:
return new ScriptDocValues.Booleans(getLongValues());
} else {
default:
return new ScriptDocValues.Longs(getLongValues());
}
}

View File

@ -96,7 +96,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple
case DOUBLE:
return new SortedNumericDoubleFieldData(reader, field);
default:
return new SortedNumericLongFieldData(reader, field, numericType == NumericType.BOOLEAN);
return new SortedNumericLongFieldData(reader, field, numericType);
}
}
@ -117,8 +117,8 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple
final LeafReader reader;
final String field;
SortedNumericLongFieldData(LeafReader reader, String field, boolean isBoolean) {
super(0L, isBoolean);
SortedNumericLongFieldData(LeafReader reader, String field, NumericType numericType) {
super(0L, numericType);
this.reader = reader;
this.field = field;
}

View File

@ -376,7 +376,7 @@ public class DateFieldMapper extends FieldMapper {
@Override
public IndexFieldData.Builder fielddataBuilder() {
failIfNoDocValues();
return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
return new DocValuesIndexFieldData.Builder().numericType(NumericType.DATE);
}
@Override

View File

@ -23,10 +23,10 @@ import org.elasticsearch.common.lucene.ScorerAware;
import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
import org.elasticsearch.script.LeafSearchScript;
import org.elasticsearch.search.aggregations.AggregationExecutionException;
import org.joda.time.ReadableInstant;
import java.lang.reflect.Array;
import java.util.Collection;
import java.util.Iterator;
/**
* {@link SortingNumericDoubleValues} implementation which is based on a script
@ -47,36 +47,49 @@ public class ScriptDoubleValues extends SortingNumericDoubleValues implements Sc
if (value == null) {
resize(0);
}
else if (value instanceof Number) {
} else if (value instanceof Number) {
resize(1);
values[0] = ((Number) value).doubleValue();
}
else if (value.getClass().isArray()) {
} else if (value instanceof ReadableInstant) {
resize(1);
values[0] = ((ReadableInstant) value).getMillis();
} else if (value.getClass().isArray()) {
resize(Array.getLength(value));
for (int i = 0; i < count(); ++i) {
values[i] = ((Number) Array.get(value, i)).doubleValue();
values[i] = toDoubleValue(Array.get(value, i));
}
}
else if (value instanceof Collection) {
} else if (value instanceof Collection) {
resize(((Collection<?>) value).size());
int i = 0;
for (Iterator<?> it = ((Collection<?>) value).iterator(); it.hasNext(); ++i) {
values[i] = ((Number) it.next()).doubleValue();
for (Object v : (Collection<?>) value) {
values[i++] = toDoubleValue(v);
}
assert i == count();
}
else {
throw new AggregationExecutionException("Unsupported script value [" + value + "]");
} else {
resize(1);
values[0] = toDoubleValue(value);
}
sort();
}
private static double toDoubleValue(Object o) {
if (o instanceof Number) {
return ((Number) o).doubleValue();
} else if (o instanceof ReadableInstant) {
// Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
return ((ReadableInstant) o).getMillis();
} else if (o instanceof Boolean) {
// We do expose boolean fields as boolean in scripts, however aggregations still expect
// that scripts return the same internal representation as regular fields, so boolean
// values in scripts need to be converted to a number, and the value formatter will
// make sure of using true/false in the key_as_string field
return ((Boolean) o).booleanValue() ? 1.0 : 0.0;
} else {
throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number, date, or boolean");
}
}
@Override
public void setScorer(Scorer scorer) {
script.setScorer(scorer);

View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.lucene.ScorerAware;
import org.elasticsearch.index.fielddata.SortingNumericDocValues;
import org.elasticsearch.script.LeafSearchScript;
import org.elasticsearch.search.aggregations.AggregationExecutionException;
import org.joda.time.ReadableInstant;
import java.lang.reflect.Array;
import java.util.Collection;
@ -77,6 +78,9 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA
private static long toLongValue(Object o) {
if (o instanceof Number) {
return ((Number) o).longValue();
} else if (o instanceof ReadableInstant) {
// Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
return ((ReadableInstant) o).getMillis();
} else if (o instanceof Boolean) {
// We do expose boolean fields as boolean in scripts, however aggregations still expect
// that scripts return the same internal representation as regular fields, so boolean
@ -84,7 +88,7 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA
// make sure of using true/false in the key_as_string field
return ((Boolean) o).booleanValue() ? 1L : 0L;
} else {
throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number");
throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number, date, or boolean");
}
}

View File

@ -62,10 +62,12 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase {
}
MappedFieldType fieldType = context.mapperService().fullName(field);
if (fieldType != null) {
/* Because this is called once per document we end up creating a new ScriptDocValues for every document which is important
* because the values inside ScriptDocValues might be reused for different documents (Dates do this). */
AtomicFieldData data = context.fieldData().getForField(fieldType).load(hitContext.readerContext());
ScriptDocValues values = data.getScriptValues();
ScriptDocValues<?> values = data.getScriptValues();
values.setNextDocId(hitContext.docId());
hitField.values().addAll(values.getValues());
hitField.values().addAll(values);
}
}
}

View File

@ -39,6 +39,8 @@ public final class ScriptFieldsFetchSubPhase implements FetchSubPhase {
return;
}
for (ScriptFieldsContext.ScriptField scriptField : context.scriptFields().fields()) {
/* Because this is called once per document we end up creating new ScriptDocValues for every document which is important because
* the values inside ScriptDocValues might be reused for different documents (Dates do this). */
LeafSearchScript leafScript;
try {
leafScript = scriptField.script().getLeafSearchScript(hitContext.readerContext());

View File

@ -80,6 +80,10 @@ public class ScriptDocValuesLongsTests extends ESTestCase {
Exception e = expectThrows(UnsupportedOperationException.class, () -> longs.getDates().add(new DateTime()));
assertEquals("doc values are unmodifiable", e.getMessage());
}
assertWarnings(
"getDate on numeric fields is deprecated. Use a date field to get dates.",
"getDates on numeric fields is deprecated. Use a date field to get dates.");
}
private Longs wrap(long[][] values) {

View File

@ -84,7 +84,7 @@ public class DateScriptMocks {
@Override
public ExecutableScript newScript(Map<String, Object> params) {
return new PlusOneMonthScript((String) params.get("fieldname"));
return new PlusOneMonthScript();
}
@Override
@ -104,14 +104,9 @@ public class DateScriptMocks {
public static class PlusOneMonthScript extends AbstractSearchScript {
public static final String NAME = "date_plus_1_month";
private String fieldname;
private Map<String, Object> vars = new HashMap<>();
public PlusOneMonthScript(String fieldname) {
this.fieldname = fieldname;
}
@Override
public void setNextVar(String name, Object value) {
vars.put(name, value);

View File

@ -45,6 +45,7 @@ import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ESIntegTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.ReadableDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@ -105,8 +106,8 @@ public class SearchFieldsIT extends ESIntegTestCase {
scripts.put("doc['date'].date.millis", vars -> {
Map<?, ?> doc = (Map) vars.get("doc");
ScriptDocValues.Longs date = (ScriptDocValues.Longs) doc.get("date");
return date.getDate().getMillis();
ScriptDocValues.Dates dates = (ScriptDocValues.Dates) doc.get("date");
return dates.getValue().getMillis();
});
scripts.put("_fields['num1'].value", vars -> fieldsScript(vars, "num1"));
@ -777,6 +778,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet();
ReadableDateTime date = new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC);
client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
.field("text_field", "foo")
.field("keyword_field", "foo")
@ -786,7 +788,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
.field("long_field", 4L)
.field("float_field", 5.0f)
.field("double_field", 6.0d)
.field("date_field", Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)))
.field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
.field("boolean_field", true)
.field("ip_field", "::1")
.endObject()).execute().actionGet();
@ -820,7 +822,7 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(0).fields().get("long_field").value(), equalTo((Object) 4L));
assertThat(searchResponse.getHits().getAt(0).fields().get("float_field").value(), equalTo((Object) 5.0));
assertThat(searchResponse.getHits().getAt(0).fields().get("double_field").value(), equalTo((Object) 6.0d));
assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) 1332374400000L));
assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo(date));
assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) true));
assertThat(searchResponse.getHits().getAt(0).fields().get("text_field").value(), equalTo("foo"));
assertThat(searchResponse.getHits().getAt(0).fields().get("keyword_field").value(), equalTo("foo"));

View File

@ -5,3 +5,10 @@
The groovy scripting language was deprecated in elasticsearch 5.0 and is now removed.
Use painless instead.
==== Date fields now return dates
`doc.some_date_field.value` now returns `ReadableDateTime`s instead of
milliseconds since epoch as a `long`. The same is true for
`doc.some_date_field[some_number]`. Use `doc.some_date_field.value.millis` to
fetch the milliseconds since epoch if you need it.

View File

@ -196,10 +196,17 @@ POST hockey/player/1/_update
[float]
[[modules-scripting-painless-dates]]
=== Regular expressions
=== Dates
Dates are a little different to work with than regular values. Here is an
example returning the year of every player's birth:
Date fields are exposed as
<<painless-api-reference-org-joda-time-ReadableDateTime, `ReadableDateTime`>>s
so they support methods like
<<painless-api-reference-org-joda-time-ReadableDateTime-getYear-0, `getYear`>>,
and
<<painless-api-reference-org-joda-time-ReadableDateTime-getDayOfWeek-0, `getDayOfWeek`>>.
To get milliseconds since epoch call
<<painless-api-reference-org-joda-time-ReadableInstant-getMillis-0, `getMillis`>>.
For example, the following returns every hockey player's birth year:
[source,js]
----------------------------------------------------------------
@ -208,7 +215,7 @@ GET hockey/_search
"script_fields": {
"birth_year": {
"script": {
"inline": "doc.born.date.year"
"inline": "doc.born.value.year"
}
}
}
@ -216,18 +223,6 @@ GET hockey/_search
----------------------------------------------------------------
// CONSOLE
The key here is that instead of indexing directly into `doc.born` like you would
a normal field you have to call `doc.born.date` to get a
<<painless-api-reference-org-joda-time-ReadableDateTime, `ReadableDateTime`>>.
From there you can call methods like
<<painless-api-reference-org-joda-time-ReadableDateTime-getYear-0, `getYear`>>,
and <<painless-api-reference-org-joda-time-ReadableDateTime-getDayOfWeek-0, `getDayOfWeek`>>.
In the example above `year` is a shortcut to `getYear()`.
If the date field is a list then `date` will always return the first date. To
access all the dates use `dates` instead of `date`.
[float]
[[modules-scripting-painless-regex]]
=== Regular expressions

View File

@ -85,6 +85,14 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsear
List getDates()
}
class org.elasticsearch.index.fielddata.ScriptDocValues.Dates -> org.elasticsearch.index.fielddata.ScriptDocValues$Dates extends List,Collection,Iterable,Object {
org.joda.time.ReadableDateTime get(int)
org.joda.time.ReadableDateTime getValue()
List getValues()
org.joda.time.ReadableDateTime getDate()
List getDates()
}
class org.elasticsearch.index.fielddata.ScriptDocValues.Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
Double get(int)
double getValue()

View File

@ -109,7 +109,7 @@ setup:
script_fields:
bar:
script:
inline: "doc.date.date.dayOfWeek"
inline: "doc.date.value.dayOfWeek"
- match: { hits.hits.0.fields.bar.0: 7}
@ -123,7 +123,7 @@ setup:
script:
inline: >
StringBuilder b = new StringBuilder();
for (def date : doc.dates.dates) {
for (def date : doc.dates) {
b.append(" ").append(date.getDayOfWeek());
}
return b.toString().trim()

View File

@ -113,8 +113,8 @@ setup:
script_fields:
field:
script:
inline: "doc['date'].get(0)"
- match: { hits.hits.0.fields.field.0: 1483272672000 }
inline: "doc.date.get(0)"
- match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
- do:
search:
@ -122,8 +122,8 @@ setup:
script_fields:
field:
script:
inline: "doc['date'].value"
- match: { hits.hits.0.fields.field.0: 1483272672000 }
inline: "doc.date.value"
- match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' }
---
"geo_point":