honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Mapper] Add `ignore_missing` option to `timestamp` 

Related to #9049.

By default, the default value for `timestamp` is `now` which means the date the document was processed by the indexing chain.

You can now reject documents which not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`):

```js
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "ignore_missing" : false
        }
    }
}
```

When you update the cluster to 1.5 or master, this index created with 1.4 we automatically migrate an index created with 1.4 to the 1.5 syntax.

Let say you have defined this in elasticsearch 1.4.x:

```js
DELETE test
PUT test
{
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}
PUT test/type/_mapping
{
  "type" : {
      "_timestamp" : {
          "enabled" : true,
          "default" : null
      }
  }
}
```

After migration, the mapping become:

```js
{
   "test": {
      "mappings": {
         "type": {
            "_timestamp": {
               "enabled": true,
               "store": false,
               "ignore_missing": false
            },
            "properties": {}
         }
      }
   }
}
```

Closes #8882.
This commit is contained in:
David Pilato 2014-12-30 17:32:50 +01:00
parent fcb0186d2e
commit fb10346953
6 changed files with 210 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
docs/reference/mapping/fields/timestamp-field.asciidoc
View File

@ -91,7 +91,7 @@ within the index request or in the `_source` document.
By default, the default value is `now` which means the date the document was processed by the indexing chain.
You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:
You can reject documents which do not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`):
[source,js]
--------------------------------------------------
@ -99,14 +99,12 @@ You can disable that default value by setting `default` to `null`. It means that
"tweet" : {
"_timestamp" : {
"enabled" : true,
"default" : null
"ignore_missing" : false
}
}
}
--------------------------------------------------
If you don't provide any timestamp value, indexation will fail.
You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:
[source,js]
@ -124,3 +122,10 @@ You can also set the default value to any date respecting <<mapping-timestamp-fi
If you don't provide any timestamp value, _timestamp will be set to this default value.
coming[1.5.0]
In elasticsearch 1.4, we allowed setting explicitly `"default":null` which is not possible anymore
as we added a new `ignore_missing`setting.
When reading an index created with elasticsearch 1.4 and using this, we automatically update it by
removing `"default": null` and setting `"ignore_missing": false`

19
src/main/java/org/elasticsearch/action/index/IndexRequest.java
View File

@ -20,12 +20,11 @@
package org.elasticsearch.action.index;
import com.google.common.base.Charsets;
import org.elasticsearch.*;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.RoutingMissingException;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.DocumentRequest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.*;
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@ -659,11 +658,13 @@ public class IndexRequest extends ShardReplicationOperationRequest<IndexRequest>
if (timestamp == null) {
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
if (mappingMd != null && mappingMd.timestamp() != null) {
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
}
if (!Strings.hasText(defaultTimestamp)) {
// If we explicitly ask to reject null timestamp
if (mappingMd.timestamp().ignoreMissing() != null && mappingMd.timestamp().ignoreMissing() == false) {
throw new TimestampParsingException("timestamp is required by mapping");
}
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
}
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis());
} else {

40
src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View File

@ -177,7 +177,7 @@ public class MappingMetaData {
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null);
private final boolean enabled;
@ -191,7 +191,9 @@ public class MappingMetaData {
private final String defaultTimestamp;
public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
private final Boolean ignoreMissing;
public Timestamp(boolean enabled, String path, String format, String defaultTimestamp, Boolean ignoreMissing) {
this.enabled = enabled;
this.path = path;
if (path == null) {
@ -202,6 +204,7 @@ public class MappingMetaData {
this.format = format;
this.dateTimeFormatter = Joda.forPattern(format);
this.defaultTimestamp = defaultTimestamp;
this.ignoreMissing = ignoreMissing;
}
public boolean enabled() {
@ -232,6 +235,10 @@ public class MappingMetaData {
return this.defaultTimestamp != null;
}
public Boolean ignoreMissing() {
return ignoreMissing;
}
public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
@ -247,6 +254,7 @@ public class MappingMetaData {
if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
if (ignoreMissing != null ? !ignoreMissing.equals(timestamp.ignoreMissing) : timestamp.ignoreMissing != null) return false;
if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;
return true;
@ -260,6 +268,7 @@ public class MappingMetaData {
result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
result = 31 * result + (ignoreMissing != null ? ignoreMissing.hashCode() : 0);
return result;
}
}
@ -278,7 +287,9 @@ public class MappingMetaData {
this.source = docMapper.mappingSource();
this.id = new Id(docMapper.idFieldMapper().path());
this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(),
docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp(),
docMapper.timestampFieldMapper().ignoreMissing());
this.hasParentField = docMapper.parentFieldMapper().active();
}
@ -344,6 +355,7 @@ public class MappingMetaData {
String path = null;
String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
Boolean ignoreMissing = null;
Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
@ -356,9 +368,11 @@ public class MappingMetaData {
format = fieldNode.toString();
} else if (fieldName.equals("default") && fieldNode != null) {
defaultTimestamp = fieldNode.toString();
} else if (fieldName.equals("ignore_missing")) {
ignoreMissing = nodeBooleanValue(fieldNode);
}
}
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
} else {
this.timestamp = Timestamp.EMPTY;
}
@ -542,6 +556,10 @@ public class MappingMetaData {
out.writeOptionalString(mappingMd.timestamp().path());
out.writeString(mappingMd.timestamp().format());
out.writeOptionalString(mappingMd.timestamp().defaultTimestamp());
// TODO Remove the test in elasticsearch 2.0.0
if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
out.writeOptionalBoolean(mappingMd.timestamp().ignoreMissing());
}
out.writeBoolean(mappingMd.hasParentField());
}
@ -579,7 +597,19 @@ public class MappingMetaData {
// routing
Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
// timestamp
final Timestamp timestamp = new Timestamp(in.readBoolean(), in.readOptionalString(), in.readString(), in.readOptionalString());
boolean enabled = in.readBoolean();
String path = in.readOptionalString();
String format = in.readString();
String defaultTimestamp = in.readOptionalString();
Boolean ignoreMissing = null;
// TODO Remove the test in elasticsearch 2.0.0
if (in.getVersion().onOrAfter(Version.V_1_5_0)) {
ignoreMissing = in.readOptionalBoolean();
}
final Timestamp timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
final boolean hasParentField = in.readBoolean();
return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
}

47
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
@ -86,6 +87,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
private boolean explicitStore = false;
private Boolean ignoreMissing = null;
public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@ -111,6 +113,11 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
return builder;
}
public Builder ignoreMissing(boolean ignoreMissing) {
this.ignoreMissing = ignoreMissing;
return builder;
}
@Override
public Builder store(boolean store) {
explicitStore = true;
@ -124,6 +131,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
fieldType.setStored(false);
}
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp,
ignoreMissing,
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
}
}
@ -133,6 +141,8 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
TimestampFieldMapper.Builder builder = timestamp();
parseField(builder, builder.name, node, parserContext);
boolean defaultSet = false;
Boolean ignoreMissing = null;
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
@ -148,10 +158,33 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
builder.dateTimeFormatter(parseDateTimeFormatter(fieldNode.toString()));
iterator.remove();
} else if (fieldName.equals("default")) {
builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
if (fieldNode == null) {
if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_4_0_Beta1) &&
parserContext.indexVersionCreated().before(Version.V_1_5_0)) {
// We are reading an index created in 1.4 with feature #7036
// `default: null` was explicitly set. We need to change this index to
// `ignore_missing: false`
builder.ignoreMissing(false);
} else {
throw new TimestampParsingException("default timestamp can not be set to null");
}
} else {
builder.defaultTimestamp(fieldNode.toString());
defaultSet = true;
}
iterator.remove();
} else if (fieldName.equals("ignore_missing")) {
ignoreMissing = nodeBooleanValue(fieldNode);
builder.ignoreMissing(ignoreMissing);
iterator.remove();
}
}
// We can not accept a default value and rejecting null values at the same time
if (defaultSet && (ignoreMissing != null && ignoreMissing == false)) {
throw new TimestampParsingException("default timestamp can not be set with ignore_missing set to false");
}
return builder;
}
}
@ -165,14 +198,16 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
private final String path;
private final String defaultTimestamp;
private final FieldType defaultFieldType;
private final Boolean ignoreMissing;
public TimestampFieldMapper(Settings indexSettings) {
this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
null, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
}
protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp,
Boolean ignoreMissing,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
@ -185,6 +220,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
this.path = path;
this.defaultTimestamp = defaultTimestamp;
this.defaultFieldType = defaultFieldType(indexSettings);
this.ignoreMissing = ignoreMissing;
}
@Override
@ -204,6 +240,10 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
return this.defaultTimestamp;
}
public Boolean ignoreMissing() {
return this.ignoreMissing;
}
public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
@ -292,6 +332,9 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
builder.field("default", defaultTimestamp);
}
if (includeDefaults || ignoreMissing != null) {
builder.field("ignore_missing", ignoreMissing);
}
if (customFieldDataSettings != null) {
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
} else if (includeDefaults) {

34
src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java
View File

@ -37,7 +37,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1");
@ -55,7 +55,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startArray("id").value("id").endArray().field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1");
@ -82,7 +82,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext("id", null, "1");
@ -100,7 +100,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext("id", "routing_value1", null);
@ -118,11 +118,11 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md1 = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
MappingMetaData md2 = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
assertThat(md1, equalTo(md2));
}
@ -131,7 +131,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "routing"),
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
MappingMetaData.ParseContext parseContext = md.createParseContext(null, null, null);
@ -146,7 +146,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@ -164,7 +164,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@ -185,7 +185,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@ -206,7 +206,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("routing", "routing_value").endObject()
@ -227,7 +227,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject()
@ -245,7 +245,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.obj0.id"),
new MappingMetaData.Routing(true, "obj1.obj2.routing"),
new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1")
@ -274,7 +274,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("obj1.id"),
new MappingMetaData.Routing(true, "obj1.routing"),
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
.startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
.startObject("obj1").field("id", "id").endObject()
@ -294,7 +294,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("field1"),
new MappingMetaData.Routing(true, "field1.field1"),
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject()
.field("aaa", "wr")
@ -317,7 +317,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id("id"),
new MappingMetaData.Routing(true, "field1.field1.field2"),
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject()
.field("aaa", "wr")
@ -340,7 +340,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
new MappingMetaData.Id(null),
new MappingMetaData.Routing(true, "field1.field2"),
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null), false);
byte[] bytes = jsonBuilder().startObject()
.field("aaa", "wr")

199
src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper.timestamp;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@ -38,12 +37,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatService;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.elasticsearch.test.ElasticsearchTestCase;
@ -56,14 +50,7 @@ import java.util.List;
import java.util.Map;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isIn;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.*;
/**
*/
@ -183,6 +170,7 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.startObject("_timestamp")
.field("enabled", "yes")
.field("path", "timestamp")
.field("ignore_missing", false)
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
@ -196,12 +184,11 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
try {
request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), notNullValue());
// We should have less than one minute (probably some ms)
long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
assertThat(delay, lessThanOrEqualTo(60000L));
} catch (TimestampParsingException e) {
assertThat(e.getDetailedMessage(), containsString("timestamp is required by mapping"));
}
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
@ -338,8 +325,8 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
assertThat(delay, lessThanOrEqualTo(60000L));
}
@Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testPathMissingShouldFail() throws Exception {
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testPathMissingWithForcedNullDefaultShouldFail() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
@ -347,6 +334,23 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.field("default", (String) null)
.endObject()
.endObject().endObject();
try {
createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
} catch (TimestampParsingException e) {
assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
}
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testPathMissingShouldFail() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("path", "timestamp")
.field("ignore_missing", false)
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
@ -358,17 +362,56 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
try {
request.process(metaData, mappingMetaData, true, "test");
fail("we should reject the mapping with a TimestampParsingException: timestamp is required by mapping");
} catch (TimestampParsingException e) {
assertThat(e.getDetailedMessage(), containsString("timestamp is required by mapping"));
}
}
@Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampMissingShouldFail() throws Exception {
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampMissingWithForcedNullDefaultShouldFail() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("default", (String) null)
.endObject()
.endObject().endObject();
try {
createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
} catch (TimestampParsingException e) {
assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
}
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampDefaultAndIgnore() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("default", "1971-12-26")
.field("ignore_missing", false)
.endObject()
.endObject().endObject();
try {
createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set with ignore_missing set to false");
} catch (TimestampParsingException e) {
assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false"));
}
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
public void testTimestampMissingShouldNotFail() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.endObject()
.endObject().endObject();
XContentBuilder doc = XContentFactory.jsonBuilder()
.startObject()
.field("foo", "bar")
@ -381,6 +424,12 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), notNullValue());
// We should have less than one minute (probably some ms)
long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
assertThat(delay, lessThanOrEqualTo(60000L));
}
@Test
@ -388,7 +437,7 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
// Testing null value for default timestamp
{
MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null,
TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, null);
TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, null, null);
MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)),
new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false);
@ -405,7 +454,24 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
// Testing "now" value for default timestamp
{
MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null,
TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, "now");
TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, "now", null);
MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)),
new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false);
BytesStreamOutput out = new BytesStreamOutput();
MappingMetaData.writeTo(expected, out);
out.close();
BytesReference bytes = out.bytes();
MappingMetaData metaData = MappingMetaData.readFrom(new BytesStreamInput(bytes));
assertThat(metaData, is(expected));
}
// Testing "ignore_missing" value for default timestamp
{
MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null,
TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, "now", false);
MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)),
new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false);
@ -551,87 +617,6 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
}
}
/**
* Test case for #9204
*/
@Test
public void testMergingNullValues() throws Exception {
// From trying to add another field with default = null
String mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("default", (String) null)
.endObject()
.endObject().endObject().string();
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
DocumentMapper docMapper = parser.parse(mapping);
mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("default", (String) null)
.endObject()
.startObject("properties")
.startObject("foo")
.field("type", "string")
.endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper.MergeResult mergeResult = docMapper.merge(parser.parse(mapping), DocumentMapper.MergeFlags.mergeFlags().simulate(true));
assertThat(mergeResult.hasConflicts(), is(false));
client().admin().indices().delete(new DeleteIndexRequest("test")).get();
// From trying to update from null to non null
mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("default", (String) null)
.endObject()
.endObject().endObject().string();
parser = createIndex("test").mapperService().documentMapperParser();
docMapper = parser.parse(mapping);
mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("default", "now")
.endObject()
.endObject().endObject().string();
mergeResult = docMapper.merge(parser.parse(mapping), DocumentMapper.MergeFlags.mergeFlags().simulate(true));
assertThat(mergeResult.hasConflicts(), is(true));
client().admin().indices().delete(new DeleteIndexRequest("test")).get();
// From trying to update from non null to null
mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("default", "now")
.endObject()
.endObject().endObject().string();
parser = createIndex("test").mapperService().documentMapperParser();
docMapper = parser.parse(mapping);
mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", true)
.field("default", (String) null)
.endObject()
.endObject().endObject().string();
mergeResult = docMapper.merge(parser.parse(mapping), DocumentMapper.MergeFlags.mergeFlags().simulate(true));
assertThat(mergeResult.hasConflicts(), is(true));
}
/**
* Test for issue #9223
*/