honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Mappings: Store _timestamp by default. 

Storing `_timestamp` by default means that under the default configuration, you
would have all the information you need in order to reindex into a different
index.

Close #8139
This commit is contained in:
Adrien Grand 2014-10-17 15:34:46 +02:00
parent a64a5c148b
commit f4ee3f25e4
8 changed files with 60 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/reference/mapping/fields/timestamp-field.asciidoc
View File

@ -24,7 +24,7 @@ should be defined:
[float]
==== store / index
By default the `_timestamp` field has `store` set to `false` and `index`
By default the `_timestamp` field has `store` set to `true` and `index`
set to `not_analyzed`. It can be queried as a standard date field.
[float]

1
rest-api-spec/test/create/70_timestamp.yaml
View File

@ -9,7 +9,6 @@
test:
_timestamp:
enabled: 1
store: yes
- do:
cluster.health:
wait_for_status: yellow

1
rest-api-spec/test/create/75_ttl.yaml
View File

@ -12,7 +12,6 @@
test:
_ttl:
enabled: 1
store: yes
default: 10s
- do:
cluster.health:

1
rest-api-spec/test/update/70_timestamp.yaml
View File

@ -9,7 +9,6 @@
test:
_timestamp:
enabled: 1
store: yes
- do:
cluster.health:
wait_for_status: yellow

2
rest-api-spec/test/update/85_fields_meta.yaml
View File

@ -14,10 +14,8 @@
_parent: { type: "foo" }
_timestamp:
enabled: 1
store: yes
_ttl:
enabled: 1
store: yes
default: 10s
- do:

28
src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View File

@ -22,12 +22,12 @@ package org.elasticsearch.index.mapper.internal;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider;
@ -58,13 +58,17 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
public static class Defaults extends DateFieldMapper.Defaults {
public static final String NAME = "_timestamp";
public static final FieldType PRE_20_FIELD_TYPE;
public static final FieldType FIELD_TYPE = new FieldType(DateFieldMapper.Defaults.FIELD_TYPE);
static {
FIELD_TYPE.setStored(false);
FIELD_TYPE.setStored(true);
FIELD_TYPE.setIndexed(true);
FIELD_TYPE.setTokenized(false);
FIELD_TYPE.freeze();
PRE_20_FIELD_TYPE = new FieldType(FIELD_TYPE);
PRE_20_FIELD_TYPE.setStored(false);
PRE_20_FIELD_TYPE.freeze();
}
public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
@ -79,6 +83,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
private String path = Defaults.PATH;
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
private boolean explicitStore = false;
public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@ -104,8 +109,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
return builder;
}
@Override
public Builder store(boolean store) {
explicitStore = true;
return super.store(store);
}
@Override
public TimestampFieldMapper build(BuilderContext context) {
if (explicitStore == false && context.indexCreatedVersion().before(Version.V_2_0_0)) {
assert fieldType.stored();
fieldType.setStored(false);
}
boolean roundCeil = Defaults.ROUND_CEIL;
if (context.indexSettings() != null) {
Settings settings = context.indexSettings();
@ -139,14 +154,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
}
}
private static FieldType defaultFieldType(Settings settings) {
return Version.indexCreated(settings).onOrAfter(Version.V_2_0_0) ? Defaults.FIELD_TYPE : Defaults.PRE_20_FIELD_TYPE;
}
private EnabledAttributeMapper enabledState;
private final String path;
private final String defaultTimestamp;
private final FieldType defaultFieldType;
public TimestampFieldMapper(Settings indexSettings) {
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
}
@ -163,11 +182,12 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
this.enabledState = enabledState;
this.path = path;
this.defaultTimestamp = defaultTimestamp;
this.defaultFieldType = defaultFieldType(indexSettings);
}
@Override
public FieldType defaultFieldType() {
return Defaults.FIELD_TYPE;
return defaultFieldType;
}
public boolean enabled() {

38
src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
View File

@ -19,8 +19,10 @@
package org.elasticsearch.index.mapper.timestamp;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.bytes.BytesReference;
@ -28,25 +30,36 @@ import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.*;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isIn;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.hamcrest.Matchers.notNullValue;
/**
*/
@ -86,13 +99,19 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
@Test
public void testDefaultValues() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
for (Version version : Arrays.asList(Version.V_1_5_0, Version.V_2_0_0, ElasticsearchTestCase.randomVersion())) {
for (String mapping : Arrays.asList(
XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(),
XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().string())) {
DocumentMapper docMapper = createIndex("test", ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build()).mapperService().documentMapperParser().parse(mapping);
assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled));
assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.stored()));
assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(version.onOrAfter(Version.V_2_0_0) ? true : false));
assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.indexed()));
assertThat(docMapper.timestampFieldMapper().path(), equalTo(TimestampFieldMapper.Defaults.PATH));
assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo(TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT));
assertAcked(client().admin().indices().prepareDelete("test").execute().get());
}
}
}
@ -100,13 +119,13 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
public void testSetValues() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp")
.field("enabled", "yes").field("store", "yes").field("index", "no")
.field("enabled", "yes").field("store", "no").field("index", "no")
.field("path", "timestamp").field("format", "year")
.endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(true));
assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(true));
assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(false));
assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(false));
assertThat(docMapper.timestampFieldMapper().path(), equalTo("timestamp"));
assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo("year"));
@ -144,8 +163,6 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
assertThat(serializedMap, hasKey("_timestamp"));
assertThat(serializedMap.get("_timestamp"), instanceOf(Map.class));
Map<String, Object> timestampConfiguration = (Map<String, Object>) serializedMap.get("_timestamp");
assertThat(timestampConfiguration, hasKey("store"));
assertThat(timestampConfiguration.get("store").toString(), is("true"));
assertThat(timestampConfiguration, hasKey("index"));
assertThat(timestampConfiguration.get("index").toString(), is("no"));
}
@ -418,7 +435,6 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true)
.field("index", randomBoolean() ? "no" : "analyzed") // default is "not_analyzed" which will be omitted when building the source again
.field("store", true)
.field("path", "foo")
.field("default", "1970-01-01")
.startObject("fielddata").field("format", "doc_values").endObject()

8
src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java
View File

@ -135,21 +135,21 @@ public class UpdateMappingOnClusterTests extends ElasticsearchIntegrationTest {
@Test
public void testUpdateTimestamp() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "yes").endObject()
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "no").endObject()
.endObject().endObject();
client().admin().indices().prepareCreate("test").addMapping("type", mapping).get();
GetMappingsResponse appliedMappings = client().admin().indices().prepareGetMappings("test").get();
LinkedHashMap timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp");
assertThat((Boolean) timestampMapping.get("store"), equalTo(true));
assertThat((Boolean) timestampMapping.get("store"), equalTo(false));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("lazy"));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("doc_values"));
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "yes").endObject()
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "no").endObject()
.endObject().endObject();
PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
appliedMappings = client().admin().indices().prepareGetMappings("test").get();
timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp");
assertThat((Boolean) timestampMapping.get("store"), equalTo(true));
assertThat((Boolean) timestampMapping.get("store"), equalTo(false));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("eager"));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("array"));
}