Merge branch 'master' into feature/query-refactoring

Conflicts:
	core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
This commit is contained in:
Christoph Büscher 2015-06-26 14:44:05 +02:00
commit 6678acfe23
16 changed files with 1018 additions and 435 deletions

View File

@ -22,6 +22,7 @@ package org.elasticsearch.action.index;
import com.google.common.base.Charsets; import com.google.common.base.Charsets;
import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.action.*; import org.elasticsearch.action.*;
import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.replication.ReplicationRequest;
import org.elasticsearch.client.Requests; import org.elasticsearch.client.Requests;
@ -562,8 +563,10 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
routing(metaData.resolveIndexRouting(routing, index)); routing(metaData.resolveIndexRouting(routing, index));
// resolve timestamp if provided externally // resolve timestamp if provided externally
if (timestamp != null) { if (timestamp != null) {
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp,
mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER); mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER,
version);
} }
// extract values if needed // extract values if needed
if (mappingMd != null) { if (mappingMd != null) {
@ -586,7 +589,8 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
if (parseContext.shouldParseTimestamp()) { if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp(); timestamp = parseContext.timestamp();
if (timestamp != null) { if (timestamp != null) {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter()); Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter(), version);
} }
} }
} catch (MapperParsingException e) { } catch (MapperParsingException e) {
@ -638,7 +642,8 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) { if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis()); timestamp = Long.toString(System.currentTimeMillis());
} else { } else {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter()); Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), version);
} }
} }
} }

View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.metadata; package org.elasticsearch.cluster.metadata;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.AbstractDiffable;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
@ -160,10 +161,22 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
public static class Timestamp { public static class Timestamp {
public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter) throws TimestampParsingException { private static final FormatDateTimeFormatter EPOCH_MILLIS_PARSER = Joda.forPattern("epoch_millis");
public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter,
Version version) throws TimestampParsingException {
try {
// no need for unix timestamp parsing in 2.x
FormatDateTimeFormatter formatter = version.onOrAfter(Version.V_2_0_0) ? dateTimeFormatter : EPOCH_MILLIS_PARSER;
return Long.toString(formatter.parser().parseMillis(timestampAsString));
} catch (RuntimeException e) {
if (version.before(Version.V_2_0_0)) {
try { try {
return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString)); return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
} catch (RuntimeException e) { } catch (RuntimeException e1) {
throw new TimestampParsingException(timestampAsString, e1);
}
}
throw new TimestampParsingException(timestampAsString, e); throw new TimestampParsingException(timestampAsString, e);
} }
} }

View File

@ -30,6 +30,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.util.ToStringUtils;
import org.elasticsearch.Version;
import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.action.fieldstats.FieldStats;
import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
@ -46,18 +47,17 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.NumericDateAnalyzer; import org.elasticsearch.index.analysis.NumericDateAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.core.LongFieldMapper.CustomLongNumericField; import org.elasticsearch.index.mapper.core.LongFieldMapper.CustomLongNumericField;
import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SearchContext;
import org.joda.time.DateTimeZone; import org.joda.time.DateTimeZone;
import java.io.IOException; import java.io.IOException;
import java.util.Iterator; import java.util.*;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -70,7 +70,7 @@ public class DateFieldMapper extends NumberFieldMapper {
public static final String CONTENT_TYPE = "date"; public static final String CONTENT_TYPE = "date";
public static class Defaults extends NumberFieldMapper.Defaults { public static class Defaults extends NumberFieldMapper.Defaults {
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime", Locale.ROOT); public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime||epoch_millis", Locale.ROOT);
public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS; public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
public static final DateFieldType FIELD_TYPE = new DateFieldType(); public static final DateFieldType FIELD_TYPE = new DateFieldType();
@ -126,6 +126,14 @@ public class DateFieldMapper extends NumberFieldMapper {
protected void setupFieldType(BuilderContext context) { protected void setupFieldType(BuilderContext context) {
FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter; FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
// TODO MOVE ME OUTSIDE OF THIS SPACE?
if (Version.indexCreated(context.indexSettings()).before(Version.V_2_0_0)) {
boolean includesEpochFormatter = dateTimeFormatter.format().contains("epoch_");
if (!includesEpochFormatter) {
String format = fieldType().timeUnit().equals(TimeUnit.SECONDS) ? "epoch_second" : "epoch_millis";
fieldType().setDateTimeFormatter(Joda.forPattern(format + "||" + dateTimeFormatter.format()));
}
}
if (!locale.equals(dateTimeFormatter.locale())) { if (!locale.equals(dateTimeFormatter.locale())) {
fieldType().setDateTimeFormatter(new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale)); fieldType().setDateTimeFormatter(new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale));
} }
@ -308,15 +316,7 @@ public class DateFieldMapper extends NumberFieldMapper {
} }
protected long parseStringValue(String value) { protected long parseStringValue(String value) {
try {
return dateTimeFormatter().parser().parseMillis(value); return dateTimeFormatter().parser().parseMillis(value);
} catch (RuntimeException e) {
try {
return timeUnit().toMillis(Long.parseLong(value));
} catch (NumberFormatException e1) {
throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter().format() + "], and timestamp number with locale [" + dateTimeFormatter().locale() + "]", e);
}
}
} }
@Override @Override

View File

@ -147,6 +147,9 @@ public class RootObjectMapper extends ObjectMapper {
List<FormatDateTimeFormatter> dateTimeFormatters = newArrayList(); List<FormatDateTimeFormatter> dateTimeFormatters = newArrayList();
if (fieldNode instanceof List) { if (fieldNode instanceof List) {
for (Object node1 : (List) fieldNode) { for (Object node1 : (List) fieldNode) {
if (node1.toString().startsWith("epoch_")) {
throw new MapperParsingException("Epoch ["+ node1.toString() +"] is not supported as dynamic date format");
}
dateTimeFormatters.add(parseDateTimeFormatter(node1)); dateTimeFormatters.add(parseDateTimeFormatter(node1));
} }
} else if ("none".equals(fieldNode.toString())) { } else if ("none".equals(fieldNode.toString())) {

View File

@ -181,7 +181,7 @@ public class SimpleCountTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testThatNonEpochDatesCanBeSearch() throws Exception { public void testThatNonEpochDatesCanBeSearched() throws Exception {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.addMapping("type1", .addMapping("type1",
jsonBuilder().startObject().startObject("type1") jsonBuilder().startObject().startObject("type1")
@ -201,16 +201,9 @@ public class SimpleCountTests extends ElasticsearchIntegrationTest {
.endObject(); .endObject();
assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true)); assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true));
// this is a timestamp in 2015 and should not be returned in counting when filtering by year
document = jsonBuilder()
.startObject()
.field("date_field", "1433236702")
.endObject();
assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true));
refresh(); refresh();
assertHitCount(client().prepareCount("test").get(), 3); assertHitCount(client().prepareCount("test").get(), 2);
CountResponse countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("date_field").from("2015010100").to("2015123123")).get(); CountResponse countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("date_field").from("2015010100").to("2015123123")).get();
assertHitCount(countResponse, 1); assertHitCount(countResponse, 1);

View File

@ -0,0 +1,205 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.date;
import com.google.common.collect.Lists;
import org.elasticsearch.Version;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.junit.Before;
import java.util.List;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
import static org.hamcrest.Matchers.is;
/**
* Test class to check for all the conditions defined in
* https://github.com/elastic/elasticsearch/issues/10971
*/
public class DateBackwardsCompatibilityTests extends ElasticsearchSingleNodeTest {
private String index = "testindex";
private String type = "testtype";
private Version randomVersionBelow2x;
@Before
public void setup() throws Exception {
randomVersionBelow2x = randomVersionBetween(getRandom(), Version.V_0_90_0, Version.V_1_6_1);
}
public void testThatPre2xIndicesNumbersAreTreatedAsEpochs() throws Exception {
index = createPre2xIndexAndMapping();
long dateInMillis = 1435073872l * 1000; // Tue Jun 23 17:37:52 CEST 2015
XContentBuilder document = jsonBuilder().startObject().field("date_field", dateInMillis).endObject();
index(document);
// search for date in time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertHitCount(response, 1);
}
public void testThatPre2xFailedStringParsingLeadsToEpochParsing() throws Exception {
index = createPre2xIndexAndMapping();
long dateInMillis = 1435073872l * 1000; // Tue Jun 23 17:37:52 CEST 2015
String date = String.valueOf(dateInMillis);
XContentBuilder document = jsonBuilder().startObject().field("date_field", date).endObject();
index(document);
// search for date in time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertHitCount(response, 1);
}
public void testThatPre2xSupportsUnixTimestampsInAnyDateFormat() throws Exception {
long dateInMillis = 1435073872l * 1000; // Tue Jun 23 17:37:52 CEST 2015
List<String> dateFormats = Lists.newArrayList("dateOptionalTime", "weekDate", "tTime", "ordinalDate", "hourMinuteSecond", "hourMinute");
for (String format : dateFormats) {
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format", format).endObject()
.endObject().endObject();
index = createIndex(randomVersionBelow2x, mapping);
XContentBuilder document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", String.valueOf(dateInMillis))
.endObject();
index(document);
// indexing as regular timestamp should work as well
document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", dateInMillis)
.endObject();
index(document);
client().admin().indices().prepareDelete(index).get();
}
}
public void testThatPre2xIndicesNumbersAreTreatedAsTimestamps() throws Exception {
// looks like a unix time stamp but is meant as 2016-06-23T01:00:00.000 - see the specified date format
long date = 2015062301000l;
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format","yyyyMMddHHSSS").endObject()
.endObject().endObject();
index = createIndex(randomVersionBelow2x, mapping);
XContentBuilder document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", randomBoolean() ? String.valueOf(date) : date)
.endObject();
index(document);
// no results in expected time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24").format("dateOptionalTime");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertNoSearchHits(response);
// result in unix timestamp range
QueryBuilder timestampQuery = QueryBuilders.rangeQuery("date_field").from(2015062300000L).to(2015062302000L);
assertHitCount(client().prepareSearch(index).setQuery(timestampQuery).get(), 1);
// result should also work with regular specified dates
QueryBuilder regularTimeQuery = QueryBuilders.rangeQuery("date_field").from("2033-11-08").to("2033-11-09").format("dateOptionalTime");
assertHitCount(client().prepareSearch(index).setQuery(regularTimeQuery).get(), 1);
}
public void testThatPost2xIndicesNumbersAreTreatedAsStrings() throws Exception {
// looks like a unix time stamp but is meant as 2016-06-23T01:00:00.000 - see the specified date format
long date = 2015062301000l;
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format","yyyyMMddHHSSS").endObject()
.endObject().endObject();
index = createIndex(Version.CURRENT, mapping);
XContentBuilder document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", String.valueOf(date))
.endObject();
index(document);
document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", date)
.endObject();
index(document);
// search for date in time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24").format("dateOptionalTime");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertHitCount(response, 2);
}
public void testDynamicDateDetectionIn2xDoesNotSupportEpochs() throws Exception {
try {
XContentBuilder mapping = jsonBuilder().startObject()
.startArray("dynamic_date_formats").value("dateOptionalTime").value("epoch_seconds").endArray()
.endObject();
createIndex(Version.CURRENT, mapping);
fail("Expected a MapperParsingException, but did not happen");
} catch (MapperParsingException e) {
assertThat(e.getMessage(), is("mapping [" + type + "]"));
}
}
private String createPre2xIndexAndMapping() throws Exception {
return createIndexAndMapping(randomVersionBelow2x);
}
private String createIndexAndMapping(Version version) throws Exception {
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format", "dateOptionalTime").endObject()
.endObject().endObject();
return createIndex(version, mapping);
}
private String createIndex(Version version, XContentBuilder mapping) {
Settings settings = settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
createIndex(index, settings, type, mapping);
ensureGreen(index);
return index;
}
private void index(XContentBuilder document) {
IndexResponse indexResponse = client().prepareIndex(index, type).setSource(document).setRefresh(true).get();
assertThat(indexResponse.isCreated(), is(true));
}
}

View File

@ -25,7 +25,9 @@ import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.util.Constants; import org.apache.lucene.util.Constants;
import org.elasticsearch.Version;
import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.common.util.LocaleUtils;
@ -53,6 +55,7 @@ import java.util.*;
import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.index.mapper.string.SimpleStringMappingTests.docValuesType; import static org.elasticsearch.index.mapper.string.SimpleStringMappingTests.docValuesType;
import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
import static org.hamcrest.Matchers.*; import static org.hamcrest.Matchers.*;
public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest { public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
@ -147,8 +150,17 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
int i = 0; int i = 0;
private DocumentMapper mapper(String type, String mapping) throws IOException { private DocumentMapper mapper(String type, String mapping) throws IOException {
return mapper(type, mapping, Version.CURRENT);
}
private DocumentMapper mapper(String type, String mapping, Version version) throws IOException {
final String indexName = "test-" + (i++); final String indexName = "test-" + (i++);
IndexService index = createIndex(indexName); IndexService index;
if (version.equals(Version.CURRENT)) {
index = createIndex(indexName);
} else {
index = createIndex(indexName, settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build());
}
client().admin().indices().preparePutMapping(indexName).setType(type).setSource(mapping).get(); client().admin().indices().preparePutMapping(indexName).setType(type).setSource(mapping).get();
return index.mapperService().documentMapper(type); return index.mapperService().documentMapper(type);
} }
@ -181,8 +193,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject().endObject().string(); .endObject().endObject().string();
DocumentMapper defaultMapper = mapper("type", mapping); DocumentMapper defaultMapper = mapper("type", mapping);
long value = System.currentTimeMillis(); long value = System.currentTimeMillis();
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject() .startObject()
.field("date_field", value) .field("date_field", value)
@ -290,7 +302,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject() .endObject()
.bytes()); .bytes());
} catch (MapperParsingException e) { } catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class)); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), is("failed to parse [field2]"));
} }
// Verify that the default is false // Verify that the default is false
@ -301,7 +314,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject() .endObject()
.bytes()); .bytes());
} catch (MapperParsingException e) { } catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class)); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), is("failed to parse [field3]"));
} }
// Unless the global ignore_malformed option is set to true // Unless the global ignore_malformed option is set to true
@ -322,7 +336,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject() .endObject()
.bytes()); .bytes());
} catch (MapperParsingException e) { } catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class)); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), is("failed to parse [field2]"));
} }
} }
@ -399,12 +414,12 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
throw new AssertionError("missing"); throw new AssertionError("missing");
} }
public void testNumericResolution() throws Exception { public void testNumericResolutionBackwardsCompat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("date_field").field("type", "date").field("format", "date_time").field("numeric_resolution", "seconds").endObject().endObject() .startObject("properties").startObject("date_field").field("type", "date").field("format", "date_time").field("numeric_resolution", "seconds").endObject().endObject()
.endObject().endObject().string(); .endObject().endObject().string();
DocumentMapper defaultMapper = mapper("type", mapping); DocumentMapper defaultMapper = mapper("type", mapping, Version.V_0_90_0);
// provided as an int // provided as an int
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
@ -429,6 +444,16 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject() .endObject()
.bytes()); .bytes());
assertThat(getDateAsMillis(doc.rootDoc(), "date_field"), equalTo(44000L)); assertThat(getDateAsMillis(doc.rootDoc(), "date_field"), equalTo(44000L));
// expected to fail due to field epoch date formatters not being set
DocumentMapper currentMapper = mapper("type", mapping);
try {
currentMapper.parse("type", "2", XContentFactory.jsonBuilder()
.startObject()
.field("date_field", randomBoolean() ? "43" : 43)
.endObject()
.bytes());
} catch (MapperParsingException e) {}
} }
public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception { public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception {

View File

@ -38,12 +38,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.Index; import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MergeResult;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.test.ElasticsearchSingleNodeTest; import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.junit.Test; import org.junit.Test;
@ -57,7 +52,9 @@ import java.util.Map;
import static org.elasticsearch.Version.V_1_5_0; import static org.elasticsearch.Version.V_1_5_0;
import static org.elasticsearch.Version.V_2_0_0; import static org.elasticsearch.Version.V_2_0_0;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.elasticsearch.test.VersionUtils.randomVersion;
import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
@ -249,15 +246,15 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.field("foo", "bar") .field("foo", "bar")
.endObject(); .endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping.string()); DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping.string());
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper); MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc); IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, mappingMetaData, true, "test"); request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), notNullValue()); assertThat(request.timestamp(), notNullValue());
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd")))); assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"), Version.CURRENT)));
} }
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
@ -274,15 +271,15 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.field("foo", "bar") .field("foo", "bar")
.endObject(); .endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper); MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc); IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, mappingMetaData, true, "test"); request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), notNullValue()); assertThat(request.timestamp(), notNullValue());
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd")))); assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"), Version.CURRENT)));
} }
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
@ -751,11 +748,12 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.startObject("_timestamp").field("enabled", true).field("path", "custom_timestamp").endObject() .startObject("_timestamp").field("enabled", true).field("path", "custom_timestamp").endObject()
.endObject().endObject().string(); .endObject().endObject().string();
DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping); DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("custom_timestamp", 1).endObject(); XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("custom_timestamp", 1).endObject();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper); MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc); IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(MetaData.builder().build(), mappingMetaData, true, "test"); request.process(metaData, mappingMetaData, true, "test");
assertEquals(request.timestamp(), "1"); assertEquals(request.timestamp(), "1");
} }
@ -766,14 +764,15 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.endObject().endObject().string(); .endObject().endObject().string();
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
DocumentMapper docMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping); DocumentMapper docMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("_timestamp", 2000000).endObject(); XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("_timestamp", 2000000).endObject();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper); MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc); IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(MetaData.builder().build(), mappingMetaData, true, "test"); request.process(metaData, mappingMetaData, true, "test");
// _timestamp in a document never worked, so backcompat is ignoring the field // _timestamp in a document never worked, so backcompat is ignoring the field
assertEquals(MappingMetaData.Timestamp.parseStringTimestamp("1970", Joda.forPattern("YYYY")), request.timestamp()); assertEquals(MappingMetaData.Timestamp.parseStringTimestamp("1970", Joda.forPattern("YYYY"), Version.V_1_4_2), request.timestamp());
assertNull(docMapper.parse("type", "1", doc.bytes()).rootDoc().get("_timestamp")); assertNull(docMapper.parse("type", "1", doc.bytes()).rootDoc().get("_timestamp"));
} }
@ -782,12 +781,52 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject() .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject()
.endObject().endObject().string(); .endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject(); XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject();
IndexRequest request = new IndexRequest("test", "type", "1").source(doc).timestamp("2015060210"); IndexRequest request = new IndexRequest("test", "type", "1").source(doc).timestamp("2015060210");
MappingMetaData mappingMetaData = new MappingMetaData(docMapper); MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
request.process(MetaData.builder().build(), mappingMetaData, true, "test"); request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), is("1433239200000")); assertThat(request.timestamp(), is("1433239200000"));
} }
public void testThatIndicesBefore2xMustSupportUnixTimestampsInAnyDateFormat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true).field("format", "dateOptionalTime").endObject()
.endObject().endObject().string();
BytesReference source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
//
// test with older versions
Settings oldSettings = settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, randomVersionBetween(random(), Version.V_0_90_0, Version.V_1_6_0)).build();
DocumentMapper docMapper = createIndex("old-index", oldSettings).mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
// both index request are successfully processed
IndexRequest oldIndexDateIndexRequest = new IndexRequest("old-index", "type", "1").source(source).timestamp("1970-01-01");
oldIndexDateIndexRequest.process(metaData, new MappingMetaData(docMapper), true, "old-index");
IndexRequest oldIndexTimestampIndexRequest = new IndexRequest("old-index", "type", "1").source(source).timestamp("1234567890");
oldIndexTimestampIndexRequest.process(metaData, new MappingMetaData(docMapper), true, "old-index");
//
// test with 2.x
DocumentMapper currentMapper = createIndex("new-index").mapperService().documentMapperParser().parse(mapping);
MetaData newMetaData = client().admin().cluster().prepareState().get().getState().getMetaData();
// this works with 2.x
IndexRequest request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1970-01-01");
request.process(newMetaData, new MappingMetaData(currentMapper), true, "new-index");
// this fails with 2.x
request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1234567890");
try {
request.process(newMetaData, new MappingMetaData(currentMapper), true, "new-index");
} catch (Exception e) {
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), containsString("failed to parse timestamp [1234567890]"));
}
}
} }

View File

@ -1813,7 +1813,7 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
@Test @Test
public void testPercolatorQueryWithNowRange() throws Exception { public void testPercolatorQueryWithNowRange() throws Exception {
client().admin().indices().prepareCreate("test") client().admin().indices().prepareCreate("test")
.addMapping("my-type", "timestamp", "type=date") .addMapping("my-type", "timestamp", "type=date,format=epoch_millis")
.get(); .get();
ensureGreen(); ensureGreen();

View File

@ -449,7 +449,7 @@ public class DecayFunctionScoreTests extends ElasticsearchIntegrationTest {
assertAcked(prepareCreate("test").addMapping( assertAcked(prepareCreate("test").addMapping(
"type1", "type1",
jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("test").field("type", "string") jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("test").field("type", "string")
.endObject().startObject("num1").field("type", "date").endObject().endObject().endObject().endObject())); .endObject().startObject("num1").field("type", "date").field("format", "epoch_millis").endObject().endObject().endObject().endObject()));
ensureYellow(); ensureYellow();
client().index( client().index(
indexRequest("test").type("type1").id("1") indexRequest("test").type("type1").id("1")

File diff suppressed because it is too large Load Diff

View File

@ -58,7 +58,7 @@ Response:
[[date-format-pattern]] [[date-format-pattern]]
==== Date Format/Pattern ==== Date Format/Pattern
NOTE: this information was copied from http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html[JodaDate] NOTE: this information was copied from http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[JodaDate]
All ASCII letters are reserved as format pattern letters, which are defined as follows: All ASCII letters are reserved as format pattern letters, which are defined as follows:

View File

@ -8,9 +8,9 @@ specifying `dynamic_date_formats` in the `root object` mapping (which will
be used unless explicitly overridden by a `date` type). There are built in be used unless explicitly overridden by a `date` type). There are built in
formats supported, as well as complete custom one. formats supported, as well as complete custom one.
The parsing of dates uses http://joda-time.sourceforge.net/[Joda]. The The parsing of dates uses http://www.joda.org/joda-time/[Joda]. The
default date parsing used if no format is specified is default date parsing used if no format is specified is
http://joda-time.sourceforge.net/api-release/org/joda/time/format/ISODateTimeFormat.html#dateOptionalTimeParser()[ISODateTimeFormat.dateOptionalTimeParser]. http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateOptionalTimeParser--[ISODateTimeFormat.dateOptionalTimeParser].
An extension to the format allow to define several formats using `||` An extension to the format allow to define several formats using `||`
separator. This allows to define less strict formats that can be used, separator. This allows to define less strict formats that can be used,
@ -215,4 +215,4 @@ date formatter in that case.
=== Custom Format === Custom Format
Allows for a completely customizable date format explained Allows for a completely customizable date format explained
http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html[here]. http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].

View File

@ -378,9 +378,6 @@ defaults to `true` or to the parent `object` type setting.
|`ignore_malformed` |Ignored a malformed number. Defaults to `false`. |`ignore_malformed` |Ignored a malformed number. Defaults to `false`.
|`numeric_resolution` |The unit to use when passed in a numeric values. Possible
values include `seconds` and `milliseconds` (default).
|======================================================================= |=======================================================================
[float] [float]

View File

@ -312,6 +312,9 @@ date.
This is not supported anymore. If you want to store unix timestamps, you need to specify This is not supported anymore. If you want to store unix timestamps, you need to specify
the appropriate formats in the mapping, namely `epoch_second` or `epoch_millis`. the appropriate formats in the mapping, namely `epoch_second` or `epoch_millis`.
In addition the `numeric_resolution` mapping parameter is ignored. Use the
`epoch_second` and `epoch_millis` date formats instead.
==== Source field limitations ==== Source field limitations
The `_source` field could previously be disabled dynamically. Since this field The `_source` field could previously be disabled dynamically. Since this field
is a critical piece of many features like the Update API, it is no longer is a critical piece of many features like the Update API, it is no longer

View File

@ -80,7 +80,7 @@ providing text to a numeric field) to be ignored.
Defaults to `ROOT`. Defaults to `ROOT`.
|`time_zone` | Time Zone to be applied to any range query related to dates. See also |`time_zone` | Time Zone to be applied to any range query related to dates. See also
http://joda-time.sourceforge.net/api-release/org/joda/time/DateTimeZone.html[JODA timezone]. http://www.joda.org/joda-time/apidocs/org/joda/time/DateTimeZone.html[JODA timezone].
|======================================================================= |=======================================================================
When a multi term query is being generated, one can control how it gets When a multi term query is being generated, one can control how it gets