From f4ee3f25e40f6eded178ee736c58f54d52f8f899 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 17 Oct 2014 15:34:46 +0200 Subject: [PATCH] Mappings: Store _timestamp by default. Storing `_timestamp` by default means that under the default configuration, you would have all the information you need in order to reindex into a different index. Close #8139 --- .../mapping/fields/timestamp-field.asciidoc | 2 +- rest-api-spec/test/create/70_timestamp.yaml | 1 - rest-api-spec/test/create/75_ttl.yaml | 1 - rest-api-spec/test/update/70_timestamp.yaml | 1 - rest-api-spec/test/update/85_fields_meta.yaml | 2 - .../mapper/internal/TimestampFieldMapper.java | 28 +++++++++-- .../timestamp/TimestampMappingTests.java | 46 +++++++++++++------ .../update/UpdateMappingOnClusterTests.java | 8 ++-- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/docs/reference/mapping/fields/timestamp-field.asciidoc b/docs/reference/mapping/fields/timestamp-field.asciidoc index fcafa25f2e3..b5ab2df522b 100644 --- a/docs/reference/mapping/fields/timestamp-field.asciidoc +++ b/docs/reference/mapping/fields/timestamp-field.asciidoc @@ -24,7 +24,7 @@ should be defined: [float] ==== store / index -By default the `_timestamp` field has `store` set to `false` and `index` +By default the `_timestamp` field has `store` set to `true` and `index` set to `not_analyzed`. It can be queried as a standard date field. [float] diff --git a/rest-api-spec/test/create/70_timestamp.yaml b/rest-api-spec/test/create/70_timestamp.yaml index f48a9fa494f..b0cbc80bb84 100644 --- a/rest-api-spec/test/create/70_timestamp.yaml +++ b/rest-api-spec/test/create/70_timestamp.yaml @@ -9,7 +9,6 @@ test: _timestamp: enabled: 1 - store: yes - do: cluster.health: wait_for_status: yellow diff --git a/rest-api-spec/test/create/75_ttl.yaml b/rest-api-spec/test/create/75_ttl.yaml index 05eb88e55c8..f5ff7f5335a 100644 --- a/rest-api-spec/test/create/75_ttl.yaml +++ b/rest-api-spec/test/create/75_ttl.yaml @@ -12,7 +12,6 @@ test: _ttl: enabled: 1 - store: yes default: 10s - do: cluster.health: diff --git a/rest-api-spec/test/update/70_timestamp.yaml b/rest-api-spec/test/update/70_timestamp.yaml index 368f750425f..21cd297b4c4 100644 --- a/rest-api-spec/test/update/70_timestamp.yaml +++ b/rest-api-spec/test/update/70_timestamp.yaml @@ -9,7 +9,6 @@ test: _timestamp: enabled: 1 - store: yes - do: cluster.health: wait_for_status: yellow diff --git a/rest-api-spec/test/update/85_fields_meta.yaml b/rest-api-spec/test/update/85_fields_meta.yaml index 97cefa1e4ff..dd265cfb5fa 100644 --- a/rest-api-spec/test/update/85_fields_meta.yaml +++ b/rest-api-spec/test/update/85_fields_meta.yaml @@ -14,10 +14,8 @@ _parent: { type: "foo" } _timestamp: enabled: 1 - store: yes _ttl: enabled: 1 - store: yes default: 10s - do: diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 5be97130ade..23d13044d99 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -22,12 +22,12 @@ package org.elasticsearch.index.mapper.internal; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider; @@ -58,13 +58,17 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap public static class Defaults extends DateFieldMapper.Defaults { public static final String NAME = "_timestamp"; + public static final FieldType PRE_20_FIELD_TYPE; public static final FieldType FIELD_TYPE = new FieldType(DateFieldMapper.Defaults.FIELD_TYPE); static { - FIELD_TYPE.setStored(false); + FIELD_TYPE.setStored(true); FIELD_TYPE.setIndexed(true); FIELD_TYPE.setTokenized(false); FIELD_TYPE.freeze(); + PRE_20_FIELD_TYPE = new FieldType(FIELD_TYPE); + PRE_20_FIELD_TYPE.setStored(false); + PRE_20_FIELD_TYPE.freeze(); } public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED; @@ -79,6 +83,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap private String path = Defaults.PATH; private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER; private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP; + private boolean explicitStore = false; public Builder() { super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT); @@ -104,8 +109,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap return builder; } + @Override + public Builder store(boolean store) { + explicitStore = true; + return super.store(store); + } + @Override public TimestampFieldMapper build(BuilderContext context) { + if (explicitStore == false && context.indexCreatedVersion().before(Version.V_2_0_0)) { + assert fieldType.stored(); + fieldType.setStored(false); + } boolean roundCeil = Defaults.ROUND_CEIL; if (context.indexSettings() != null) { Settings settings = context.indexSettings(); @@ -139,14 +154,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap } } + private static FieldType defaultFieldType(Settings settings) { + return Version.indexCreated(settings).onOrAfter(Version.V_2_0_0) ? Defaults.FIELD_TYPE : Defaults.PRE_20_FIELD_TYPE; + } private EnabledAttributeMapper enabledState; private final String path; private final String defaultTimestamp; + private final FieldType defaultFieldType; public TimestampFieldMapper(Settings indexSettings) { - this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP, + this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP, Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings); } @@ -163,11 +182,12 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap this.enabledState = enabledState; this.path = path; this.defaultTimestamp = defaultTimestamp; + this.defaultFieldType = defaultFieldType(indexSettings); } @Override public FieldType defaultFieldType() { - return Defaults.FIELD_TYPE; + return defaultFieldType; } public boolean enabled() { diff --git a/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index 08d7fa1b8e8..57dfeabd231 100644 --- a/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -19,8 +19,10 @@ package org.elasticsearch.index.mapper.timestamp; +import org.elasticsearch.Version; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.bytes.BytesReference; @@ -28,25 +30,36 @@ import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.test.ElasticsearchSingleNodeTest; +import org.elasticsearch.test.ElasticsearchTestCase; import org.junit.Test; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isIn; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; /** */ @@ -86,13 +99,19 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest { @Test public void testDefaultValues() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled)); - assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.stored())); - assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.indexed())); - assertThat(docMapper.timestampFieldMapper().path(), equalTo(TimestampFieldMapper.Defaults.PATH)); - assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo(TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT)); + for (Version version : Arrays.asList(Version.V_1_5_0, Version.V_2_0_0, ElasticsearchTestCase.randomVersion())) { + for (String mapping : Arrays.asList( + XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(), + XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().string())) { + DocumentMapper docMapper = createIndex("test", ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build()).mapperService().documentMapperParser().parse(mapping); + assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled)); + assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(version.onOrAfter(Version.V_2_0_0) ? true : false)); + assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.indexed())); + assertThat(docMapper.timestampFieldMapper().path(), equalTo(TimestampFieldMapper.Defaults.PATH)); + assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo(TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT)); + assertAcked(client().admin().indices().prepareDelete("test").execute().get()); + } + } } @@ -100,13 +119,13 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest { public void testSetValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") - .field("enabled", "yes").field("store", "yes").field("index", "no") + .field("enabled", "yes").field("store", "no").field("index", "no") .field("path", "timestamp").field("format", "year") .endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(true)); - assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(true)); + assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(false)); assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(false)); assertThat(docMapper.timestampFieldMapper().path(), equalTo("timestamp")); assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo("year")); @@ -144,8 +163,6 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest { assertThat(serializedMap, hasKey("_timestamp")); assertThat(serializedMap.get("_timestamp"), instanceOf(Map.class)); Map timestampConfiguration = (Map) serializedMap.get("_timestamp"); - assertThat(timestampConfiguration, hasKey("store")); - assertThat(timestampConfiguration.get("store").toString(), is("true")); assertThat(timestampConfiguration, hasKey("index")); assertThat(timestampConfiguration.get("index").toString(), is("no")); } @@ -418,7 +435,6 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true) .field("index", randomBoolean() ? "no" : "analyzed") // default is "not_analyzed" which will be omitted when building the source again - .field("store", true) .field("path", "foo") .field("default", "1970-01-01") .startObject("fielddata").field("format", "doc_values").endObject() diff --git a/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java b/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java index db6a5063380..f9364f39b05 100644 --- a/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java @@ -135,21 +135,21 @@ public class UpdateMappingOnClusterTests extends ElasticsearchIntegrationTest { @Test public void testUpdateTimestamp() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "no").endObject() .endObject().endObject(); client().admin().indices().prepareCreate("test").addMapping("type", mapping).get(); GetMappingsResponse appliedMappings = client().admin().indices().prepareGetMappings("test").get(); LinkedHashMap timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp"); - assertThat((Boolean) timestampMapping.get("store"), equalTo(true)); + assertThat((Boolean) timestampMapping.get("store"), equalTo(false)); assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("lazy")); assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("doc_values")); mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "no").endObject() .endObject().endObject(); PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get(); appliedMappings = client().admin().indices().prepareGetMappings("test").get(); timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp"); - assertThat((Boolean) timestampMapping.get("store"), equalTo(true)); + assertThat((Boolean) timestampMapping.get("store"), equalTo(false)); assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("eager")); assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("array")); }