diff --git a/docs/reference/mapping/fields/timestamp-field.asciidoc b/docs/reference/mapping/fields/timestamp-field.asciidoc index 20cf754e375..fcafa25f2e3 100644 --- a/docs/reference/mapping/fields/timestamp-field.asciidoc +++ b/docs/reference/mapping/fields/timestamp-field.asciidoc @@ -4,7 +4,7 @@ The `_timestamp` field allows to automatically index the timestamp of a document. It can be provided externally via the index request or in the `_source`. If it is not provided externally it will be automatically set -to the date the document was processed by the indexing chain. +to a <>. [float] ==== enabled @@ -60,6 +60,7 @@ Note, using `path` without explicit timestamp value provided requires an additional (though quite fast) parsing phase. [float] +[[mapping-timestamp-field-format]] ==== format You can define the <> used to parse the provided timestamp value. For example: Note, the default format is `dateOptionalTime`. The timestamp value will first be parsed as a number and if it fails the format will be tried. + +[float] +[[mapping-timestamp-field-default]] +==== default + +You can define a default value for when timestamp is not provided +within the index request or in the `_source` document. + +By default, the default value is `now` which means the date the document was processed by the indexing chain. + +You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory: + +[source,js] +-------------------------------------------------- +{ + "tweet" : { + "_timestamp" : { + "enabled" : true, + "default" : null + } + } +} +-------------------------------------------------- + +If you don't provide any timestamp value, indexation will fail. + +You can also set the default value to any date respecting <>: + +[source,js] +-------------------------------------------------- +{ + "tweet" : { + "_timestamp" : { + "enabled" : true, + "format" : "YYYY-MM-dd", + "default" : "1970-01-01" + } + } +} +-------------------------------------------------- + +If you don't provide any timestamp value, indexation will fail. + diff --git a/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 646f4f9413b..f6d9065d796 100644 --- a/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -23,6 +23,7 @@ import com.google.common.base.Charsets; import org.elasticsearch.*; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.RoutingMissingException; +import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -574,7 +575,9 @@ public class IndexRequest extends ShardReplicationOperationRequest } if (parseContext.shouldParseTimestamp()) { timestamp = parseContext.timestamp(); - timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter()); + if (timestamp != null) { + timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter()); + } } } catch (MapperParsingException e) { throw e; @@ -613,7 +616,18 @@ public class IndexRequest extends ShardReplicationOperationRequest // generate timestamp if not provided, we always have one post this stage... if (timestamp == null) { - timestamp = Long.toString(System.currentTimeMillis()); + String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP; + if (mappingMd != null && mappingMd.timestamp() != null) { + defaultTimestamp = mappingMd.timestamp().defaultTimestamp(); + } + if (!Strings.hasText(defaultTimestamp)) { + throw new TimestampParsingException("timestamp is required by mapping"); + } + if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) { + timestamp = Long.toString(System.currentTimeMillis()); + } else { + timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter()); + } } } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java index 9461e866800..1dde1081746 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.ElasticsearchIllegalStateException; +import org.elasticsearch.Version; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -175,7 +176,8 @@ public class MappingMetaData { } - public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT); + public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, + TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP); private final boolean enabled; @@ -187,7 +189,9 @@ public class MappingMetaData { private final FormatDateTimeFormatter dateTimeFormatter; - public Timestamp(boolean enabled, String path, String format) { + private final String defaultTimestamp; + + public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) { this.enabled = enabled; this.path = path; if (path == null) { @@ -197,6 +201,7 @@ public class MappingMetaData { } this.format = format; this.dateTimeFormatter = Joda.forPattern(format); + this.defaultTimestamp = defaultTimestamp; } public boolean enabled() { @@ -219,6 +224,14 @@ public class MappingMetaData { return this.format; } + public String defaultTimestamp() { + return this.defaultTimestamp; + } + + public boolean hasDefaultTimestamp() { + return this.defaultTimestamp != null; + } + public FormatDateTimeFormatter dateTimeFormatter() { return this.dateTimeFormatter; } @@ -233,6 +246,7 @@ public class MappingMetaData { if (enabled != timestamp.enabled) return false; if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false; if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false; + if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false; if (!Arrays.equals(pathElements, timestamp.pathElements)) return false; return true; @@ -245,6 +259,7 @@ public class MappingMetaData { result = 31 * result + (format != null ? format.hashCode() : 0); result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0); result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0); + result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0); return result; } } @@ -263,7 +278,7 @@ public class MappingMetaData { this.source = docMapper.mappingSource(); this.id = new Id(docMapper.idFieldMapper().path()); this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path()); - this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format()); + this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp()); this.hasParentField = docMapper.parentFieldMapper().active(); } @@ -328,6 +343,7 @@ public class MappingMetaData { boolean enabled = false; String path = null; String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT; + String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP; Map timestampNode = (Map) withoutType.get("_timestamp"); for (Map.Entry entry : timestampNode.entrySet()) { String fieldName = Strings.toUnderscoreCase(entry.getKey()); @@ -338,9 +354,11 @@ public class MappingMetaData { path = fieldNode.toString(); } else if (fieldName.equals("format")) { format = fieldNode.toString(); + } else if (fieldName.equals("default")) { + defaultTimestamp = fieldNode.toString(); } } - this.timestamp = new Timestamp(enabled, path, format); + this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp); } else { this.timestamp = Timestamp.EMPTY; } @@ -528,6 +546,14 @@ public class MappingMetaData { out.writeBoolean(false); } out.writeString(mappingMd.timestamp().format()); + if (out.getVersion().onOrAfter(Version.V_1_4_0)) { + if (mappingMd.timestamp().hasDefaultTimestamp()) { + out.writeBoolean(true); + out.writeString(mappingMd.timestamp().defaultTimestamp()); + } else { + out.writeBoolean(false); + } + } out.writeBoolean(mappingMd.hasParentField()); } @@ -565,7 +591,8 @@ public class MappingMetaData { // routing Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null); // timestamp - Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString()); + Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString(), + in.getVersion().onOrAfter(Version.V_1_4_0) ? (in.readBoolean() ? in.readString() : null) : TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP); final boolean hasParentField = in.readBoolean(); return new MappingMetaData(type, source, id, routing, timestamp, hasParentField); } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 7ecdc133de3..3239cac7c1b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -70,6 +70,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED; public static final String PATH = null; public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT); + public static final String DEFAULT_TIMESTAMP = "now"; } public static class Builder extends NumberFieldMapper.Builder { @@ -77,6 +78,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap private EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED; private String path = Defaults.PATH; private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER; + private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP; public Builder() { super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT); @@ -97,6 +99,11 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap return builder; } + public Builder defaultTimestamp(String defaultTimestamp) { + this.defaultTimestamp = defaultTimestamp; + return builder; + } + @Override public TimestampFieldMapper build(BuilderContext context) { boolean roundCeil = Defaults.ROUND_CEIL; @@ -104,7 +111,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap Settings settings = context.indexSettings(); roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL)); } - return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil, + return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil, ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings()); } } @@ -124,6 +131,8 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap builder.path(fieldNode.toString()); } else if (fieldName.equals("format")) { builder.dateTimeFormatter(parseDateTimeFormatter(builder.name(), fieldNode.toString())); + } else if (fieldName.equals("default")) { + builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString()); } } return builder; @@ -134,15 +143,16 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap private EnabledAttributeMapper enabledState; private final String path; + private final String defaultTimestamp; public TimestampFieldMapper() { - this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, + this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP, Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY); } protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path, - FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil, - Explicit ignoreMalformed,Explicit coerce, PostingsFormatProvider postingsProvider, + FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil, + Explicit ignoreMalformed, Explicit coerce, PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter, @@ -152,6 +162,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap indexSettings, MultiFields.empty(), null); this.enabledState = enabledState; this.path = path; + this.defaultTimestamp = defaultTimestamp; } @Override @@ -167,6 +178,10 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap return this.path; } + public String defaultTimestamp() { + return this.defaultTimestamp; + } + public FormatDateTimeFormatter dateTimeFormatter() { return this.dateTimeFormatter; } @@ -226,7 +241,8 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap // if all are defaults, no sense to write it at all if (!includeDefaults && fieldType.indexed() == Defaults.FIELD_TYPE.indexed() && customFieldDataSettings == null && fieldType.stored() == Defaults.FIELD_TYPE.stored() && enabledState == Defaults.ENABLED && path == Defaults.PATH - && dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) { + && dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format()) + && Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) { return builder; } builder.startObject(CONTENT_TYPE); @@ -246,6 +262,9 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap if (includeDefaults || !dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) { builder.field("format", dateTimeFormatter.format()); } + if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) { + builder.field("default", defaultTimestamp); + } if (customFieldDataSettings != null) { builder.field("fielddata", (Map) customFieldDataSettings.getAsMap()); } else if (includeDefaults) { diff --git a/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java b/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java index 65950a75e3a..b4369a28780 100644 --- a/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java +++ b/src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.test.ElasticsearchTestCase; import org.junit.Test; @@ -36,7 +37,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "routing"), - new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes(); MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1"); @@ -54,7 +55,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "routing"), - new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startArray("id").value("id").endArray().field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes(); MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1"); @@ -81,7 +82,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "routing"), - new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes(); MappingMetaData.ParseContext parseContext = md.createParseContext("id", null, "1"); @@ -99,7 +100,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "routing"), - new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes(); MappingMetaData.ParseContext parseContext = md.createParseContext("id", "routing_value1", null); @@ -117,11 +118,11 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md1 = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "routing"), - new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); MappingMetaData md2 = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "routing"), - new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); assertThat(md1, equalTo(md2)); } @@ -130,7 +131,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "routing"), - new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes(); MappingMetaData.ParseContext parseContext = md.createParseContext(null, null, null); @@ -145,7 +146,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("obj1.id"), new MappingMetaData.Routing(true, "obj1.routing"), - new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject() .startObject("obj1").field("id", "id").field("routing", "routing_value").endObject() @@ -163,7 +164,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("obj1.id"), new MappingMetaData.Routing(true, "obj1.routing"), - new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject() .startObject("obj1").field("id", "id").field("routing", "routing_value").endObject() @@ -184,7 +185,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("obj1.id"), new MappingMetaData.Routing(true, "obj1.routing"), - new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject() .startObject("obj1").field("id", "id").field("routing", "routing_value").endObject() @@ -205,7 +206,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("obj1.id"), new MappingMetaData.Routing(true, "obj1.routing"), - new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject() .startObject("obj1").field("routing", "routing_value").endObject() @@ -226,7 +227,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("obj1.id"), new MappingMetaData.Routing(true, "obj1.routing"), - new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject() .startObject("obj1").field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject() @@ -244,7 +245,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("obj1.obj0.id"), new MappingMetaData.Routing(true, "obj1.obj2.routing"), - new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject() .startObject("obj1") @@ -273,7 +274,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("obj1.id"), new MappingMetaData.Routing(true, "obj1.routing"), - new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2") .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject() .startObject("obj1").field("id", "id").endObject() @@ -293,7 +294,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("field1"), new MappingMetaData.Routing(true, "field1.field1"), - new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject() .field("aaa", "wr") @@ -316,7 +317,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id("id"), new MappingMetaData.Routing(true, "field1.field1.field2"), - new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject() .field("aaa", "wr") @@ -339,7 +340,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase { MappingMetaData md = new MappingMetaData("type1", new CompressedString(""), new MappingMetaData.Id(null), new MappingMetaData.Routing(true, "field1.field2"), - new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false); + new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false); byte[] bytes = jsonBuilder().startObject() .field("aaa", "wr") diff --git a/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index fcd0a12c012..60fbfe46086 100644 --- a/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -19,7 +19,15 @@ package org.elasticsearch.index.mapper.timestamp; +import org.elasticsearch.action.TimestampParsingException; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.io.stream.BytesStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -32,6 +40,7 @@ import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.test.ElasticsearchSingleNodeTest; import org.junit.Test; +import java.io.IOException; import java.util.Locale; import java.util.Map; @@ -154,4 +163,246 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest { assertThat(timestampConfiguration, hasKey("index")); assertThat(timestampConfiguration.get("index").toString(), is("no")); } -} \ No newline at end of file + + @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testPathMissingDefaultValue() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .field("path", "timestamp") + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + assertThat(request.timestamp(), notNullValue()); + + // We should have less than one minute (probably some ms) + long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp()); + assertThat(delay, lessThanOrEqualTo(60000L)); + } + + @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testTimestampDefaultValue() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + assertThat(request.timestamp(), notNullValue()); + + // We should have less than one minute (probably some ms) + long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp()); + assertThat(delay, lessThanOrEqualTo(60000L)); + } + + @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testPathMissingDefaultToEpochValue() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .field("path", "timestamp") + .field("default", "1970-01-01") + .field("format", "YYYY-MM-dd") + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + assertThat(request.timestamp(), notNullValue()); + assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd")))); + } + + @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testTimestampMissingDefaultToEpochValue() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .field("default", "1970-01-01") + .field("format", "YYYY-MM-dd") + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + assertThat(request.timestamp(), notNullValue()); + assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd")))); + } + + @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testPathMissingNowDefaultValue() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .field("path", "timestamp") + .field("default", "now") + .field("format", "YYYY-MM-dd") + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + assertThat(request.timestamp(), notNullValue()); + + // We should have less than one minute (probably some ms) + long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp()); + assertThat(delay, lessThanOrEqualTo(60000L)); + } + + @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testTimestampMissingNowDefaultValue() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .field("default", "now") + .field("format", "YYYY-MM-dd") + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + assertThat(request.timestamp(), notNullValue()); + + // We should have less than one minute (probably some ms) + long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp()); + assertThat(delay, lessThanOrEqualTo(60000L)); + } + + @Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testPathMissingShouldFail() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .field("path", "timestamp") + .field("default", (String) null) + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + } + + @Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] + public void testTimestampMissingShouldFail() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp") + .field("enabled", "yes") + .field("default", (String) null) + .endObject() + .endObject().endObject(); + XContentBuilder doc = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject(); + + MetaData metaData = MetaData.builder().build(); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + + MappingMetaData mappingMetaData = new MappingMetaData(docMapper); + + IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + request.process(metaData, null, mappingMetaData, true); + } + + public void testDefaultTimestampStream() throws IOException { + // Testing null value for default timestamp + { + MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null, + TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, null); + MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)), + new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false); + + BytesStreamOutput out = new BytesStreamOutput(); + MappingMetaData.writeTo(expected, out); + out.close(); + BytesReference bytes = out.bytes(); + + MappingMetaData metaData = MappingMetaData.readFrom(new BytesStreamInput(bytes)); + + assertThat(metaData, is(expected)); + } + + // Testing "now" value for default timestamp + { + MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null, + TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, "now"); + MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)), + new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false); + + BytesStreamOutput out = new BytesStreamOutput(); + MappingMetaData.writeTo(expected, out); + out.close(); + BytesReference bytes = out.bytes(); + + MappingMetaData metaData = MappingMetaData.readFrom(new BytesStreamInput(bytes)); + + assertThat(metaData, is(expected)); + } + } +}