honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Generate timestamp when path is null 

Index process fails when having `_timestamp` enabled and `path` option is set.
It fails with a `TimestampParsingException[failed to parse timestamp [null]]` message.

Reproduction:

```
DELETE test
PUT  test
{
    "mappings": {
        "test": {
            "_timestamp" : {
                "enabled" : "yes",
                "path" : "post_date"
            }
        }
    }
}
PUT test/test/1
{
  "foo": "bar"
}
```

You can define a default value for when timestamp is not provided
within the index request or in the `_source` document.

By default, the default value is `now` which means the date the document was processed by the indexing chain.

You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:

```
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "default" : null
        }
    }
}
```

If you don't provide any timestamp value, indexation will fail.

You can also set the default value to any date respecting timestamp format:

```
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "format" : "YYYY-MM-dd",
            "default" : "1970-01-01"
        }
    }
}
```

If you don't provide any timestamp value, indexation will fail.

Closes #4718.
Closes #7036.
This commit is contained in:
David Pilato 2014-07-25 15:14:31 +02:00
parent 6b39aa615e
commit 85eb0ea0e7
6 changed files with 387 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
docs/reference/mapping/fields/timestamp-field.asciidoc
View File

@ -4,7 +4,7 @@
The `_timestamp` field allows to automatically index the timestamp of a
document. It can be provided externally via the index request or in the
`_source`. If it is not provided externally it will be automatically set
to the date the document was processed by the indexing chain.
to a <<mapping-timestamp-field-default,default date>>.
[float]
==== enabled
@ -60,6 +60,7 @@ Note, using `path` without explicit timestamp value provided requires an
additional (though quite fast) parsing phase.
[float]
[[mapping-timestamp-field-format]]
==== format
You can define the <<mapping-date-format,date
@ -80,3 +81,46 @@ format>> used to parse the provided timestamp value. For example:
Note, the default format is `dateOptionalTime`. The timestamp value will
first be parsed as a number and if it fails the format will be tried.
[float]
[[mapping-timestamp-field-default]]
==== default
You can define a default value for when timestamp is not provided
within the index request or in the `_source` document.
By default, the default value is `now` which means the date the document was processed by the indexing chain.
You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:
[source,js]
--------------------------------------------------
{
"tweet" : {
"_timestamp" : {
"enabled" : true,
"default" : null
}
}
}
--------------------------------------------------
If you don't provide any timestamp value, indexation will fail.
You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:
[source,js]
--------------------------------------------------
{
"tweet" : {
"_timestamp" : {
"enabled" : true,
"format" : "YYYY-MM-dd",
"default" : "1970-01-01"
}
}
}
--------------------------------------------------
If you don't provide any timestamp value, indexation will fail.

14
src/main/java/org/elasticsearch/action/index/IndexRequest.java
View File

@ -23,6 +23,7 @@ import com.google.common.base.Charsets;
import org.elasticsearch.*;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.RoutingMissingException;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@ -574,8 +575,10 @@ public class IndexRequest extends ShardReplicationOperationRequest<IndexRequest>
}
if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp();
if (timestamp != null) {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
}
}
} catch (MapperParsingException e) {
throw e;
} catch (Exception e) {
@ -613,7 +616,18 @@ public class IndexRequest extends ShardReplicationOperationRequest<IndexRequest>
// generate timestamp if not provided, we always have one post this stage...
if (timestamp == null) {
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
if (mappingMd != null && mappingMd.timestamp() != null) {
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
}
if (!Strings.hasText(defaultTimestamp)) {
throw new TimestampParsingException("timestamp is required by mapping");
}
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis());
} else {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
}
}
}

37
src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
@ -175,7 +176,8 @@ public class MappingMetaData {
}
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT);
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
private final boolean enabled;
@ -187,7 +189,9 @@ public class MappingMetaData {
private final FormatDateTimeFormatter dateTimeFormatter;
public Timestamp(boolean enabled, String path, String format) {
private final String defaultTimestamp;
public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
this.enabled = enabled;
this.path = path;
if (path == null) {
@ -197,6 +201,7 @@ public class MappingMetaData {
}
this.format = format;
this.dateTimeFormatter = Joda.forPattern(format);
this.defaultTimestamp = defaultTimestamp;
}
public boolean enabled() {
@ -219,6 +224,14 @@ public class MappingMetaData {
return this.format;
}
public String defaultTimestamp() {
return this.defaultTimestamp;
}
public boolean hasDefaultTimestamp() {
return this.defaultTimestamp != null;
}
public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
@ -233,6 +246,7 @@ public class MappingMetaData {
if (enabled != timestamp.enabled) return false;
if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;
return true;
@ -245,6 +259,7 @@ public class MappingMetaData {
result = 31 * result + (format != null ? format.hashCode() : 0);
result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
return result;
}
}
@ -263,7 +278,7 @@ public class MappingMetaData {
this.source = docMapper.mappingSource();
this.id = new Id(docMapper.idFieldMapper().path());
this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
this.hasParentField = docMapper.parentFieldMapper().active();
}
@ -328,6 +343,7 @@ public class MappingMetaData {
boolean enabled = false;
String path = null;
String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
@ -338,9 +354,11 @@ public class MappingMetaData {
path = fieldNode.toString();
} else if (fieldName.equals("format")) {
format = fieldNode.toString();
} else if (fieldName.equals("default")) {
defaultTimestamp = fieldNode.toString();
}
}
this.timestamp = new Timestamp(enabled, path, format);
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
} else {
this.timestamp = Timestamp.EMPTY;
}
@ -528,6 +546,14 @@ public class MappingMetaData {
out.writeBoolean(false);
}
out.writeString(mappingMd.timestamp().format());
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
if (mappingMd.timestamp().hasDefaultTimestamp()) {
out.writeBoolean(true);
out.writeString(mappingMd.timestamp().defaultTimestamp());
} else {
out.writeBoolean(false);
}
}
out.writeBoolean(mappingMd.hasParentField());
}
@ -565,7 +591,8 @@ public class MappingMetaData {
// routing
Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
// timestamp
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString());
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString(),
in.getVersion().onOrAfter(Version.V_1_4_0) ? (in.readBoolean() ? in.readString() : null) : TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
final boolean hasParentField = in.readBoolean();
return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
}

29
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -70,6 +70,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
public static final String PATH = null;
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
public static final String DEFAULT_TIMESTAMP = "now";
}
public static class Builder extends NumberFieldMapper.Builder<Builder, TimestampFieldMapper> {
@ -77,6 +78,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
private EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED;
private String path = Defaults.PATH;
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@ -97,6 +99,11 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
return builder;
}
public Builder defaultTimestamp(String defaultTimestamp) {
this.defaultTimestamp = defaultTimestamp;
return builder;
}
@Override
public TimestampFieldMapper build(BuilderContext context) {
boolean roundCeil = Defaults.ROUND_CEIL;
@ -104,7 +111,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
Settings settings = context.indexSettings();
roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
}
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil,
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
}
}
@ -124,6 +131,8 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
builder.path(fieldNode.toString());
} else if (fieldName.equals("format")) {
builder.dateTimeFormatter(parseDateTimeFormatter(builder.name(), fieldNode.toString()));
} else if (fieldName.equals("default")) {
builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
}
}
return builder;
@ -134,15 +143,16 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
private EnabledAttributeMapper enabledState;
private final String path;
private final String defaultTimestamp;
public TimestampFieldMapper() {
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER,
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY);
}
protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil,
Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
@ -152,6 +162,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
indexSettings, MultiFields.empty(), null);
this.enabledState = enabledState;
this.path = path;
this.defaultTimestamp = defaultTimestamp;
}
@Override
@ -167,6 +178,10 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
return this.path;
}
public String defaultTimestamp() {
return this.defaultTimestamp;
}
public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
@ -226,7 +241,8 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
// if all are defaults, no sense to write it at all
if (!includeDefaults && fieldType.indexed() == Defaults.FIELD_TYPE.indexed() && customFieldDataSettings == null &&
fieldType.stored() == Defaults.FIELD_TYPE.stored() && enabledState == Defaults.ENABLED && path == Defaults.PATH
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())
&& Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
return builder;
}
builder.startObject(CONTENT_TYPE);
@ -246,6 +262,9 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
if (includeDefaults || !dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
builder.field("format", dateTimeFormatter.format());
}
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
builder.field("default", defaultTimestamp);
}
if (customFieldDataSettings != null) {
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
} else if (includeDefaults) {

35
src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.cluster.metadata;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
@ -36,7 +37,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1");
@ -54,7 +55,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startArray("id").value("id").endArray().field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1");
@ -81,7 +82,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext("id", null, "1");
@ -99,7 +100,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext("id", "routing_value1", null);
@ -117,11 +118,11 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md1 = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
MappingMetaData md2 = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
assertThat(md1, equalTo(md2));
}
@ -130,7 +131,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null, null);
@ -145,7 +146,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@ -163,7 +164,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@ -184,7 +185,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@ -205,7 +206,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("routing", "routing_value").endObject()
@ -226,7 +227,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject()
@ -244,7 +245,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.obj0.id"),
new MappingMetaData.Routing(true, "obj1.obj2.routing"),
new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1")
@ -273,7 +274,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").endObject()
@ -293,7 +294,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("field1"),
new MappingMetaData.Routing(true, "field1.field1"),
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject()
.field("aaa", "wr")
@ -316,7 +317,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "field1.field1.field2"),
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject()
.field("aaa", "wr")
@ -339,7 +340,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id(null),
new MappingMetaData.Routing(true, "field1.field2"),
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false);
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
byte[] bytes = jsonBuilder().startObject()
.field("aaa", "wr")

251
src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
View File

@ -19,7 +19,15 @@
package org.elasticsearch.index.mapper.timestamp;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
@ -32,6 +40,7 @@ import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.junit.Test;
import java.io.IOException;
import java.util.Locale;
import java.util.Map;
@ -154,4 +163,246 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
assertThat(timestampConfiguration, hasKey("index"));
assertThat(timestampConfiguration.get("index").toString(), is("no"));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testPathMissingDefaultValue() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("path", "timestamp")
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
assertThat(request.timestamp(), notNullValue());
// We should have less than one minute (probably some ms)
long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
assertThat(delay, lessThanOrEqualTo(60000L));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampDefaultValue() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
assertThat(request.timestamp(), notNullValue());
// We should have less than one minute (probably some ms)
long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
assertThat(delay, lessThanOrEqualTo(60000L));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testPathMissingDefaultToEpochValue() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("path", "timestamp")
.field("default", "1970-01-01")
.field("format", "YYYY-MM-dd")
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
assertThat(request.timestamp(), notNullValue());
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"))));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampMissingDefaultToEpochValue() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("default", "1970-01-01")
.field("format", "YYYY-MM-dd")
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
assertThat(request.timestamp(), notNullValue());
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"))));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testPathMissingNowDefaultValue() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("path", "timestamp")
.field("default", "now")
.field("format", "YYYY-MM-dd")
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
assertThat(request.timestamp(), notNullValue());
// We should have less than one minute (probably some ms)
long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
assertThat(delay, lessThanOrEqualTo(60000L));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampMissingNowDefaultValue() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("default", "now")
.field("format", "YYYY-MM-dd")
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
assertThat(request.timestamp(), notNullValue());
// We should have less than one minute (probably some ms)
long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
assertThat(delay, lessThanOrEqualTo(60000L));
}
@Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testPathMissingShouldFail() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("path", "timestamp")
.field("default", (String) null)
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
}
@Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampMissingShouldFail() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("default", (String) null)
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, null, mappingMetaData, true);
}
public void testDefaultTimestampStream() throws IOException {
// Testing null value for default timestamp
{
MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null,
TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, null);
MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)),
new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false);
BytesStreamOutput out = new BytesStreamOutput();
MappingMetaData.writeTo(expected, out);
out.close();
BytesReference bytes = out.bytes();
MappingMetaData metaData = MappingMetaData.readFrom(new BytesStreamInput(bytes));
assertThat(metaData, is(expected));
}
// Testing "now" value for default timestamp
{
MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null,
TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, "now");
MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)),
new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false);
BytesStreamOutput out = new BytesStreamOutput();
MappingMetaData.writeTo(expected, out);
out.close();
BytesReference bytes = out.bytes();
MappingMetaData metaData = MappingMetaData.readFrom(new BytesStreamInput(bytes));
assertThat(metaData, is(expected));
}
}
}