honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/query-refactoring 

Conflicts:
	core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
This commit is contained in:
Christoph Büscher 2015-06-26 14:44:05 +02:00
parent 9d47e809c6 f5f73259e4
commit 6678acfe23
16 changed files with 1018 additions and 435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.action.index;
import com.google.common.base.Charsets;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.action.*;
import org.elasticsearch.action.support.replication.ReplicationRequest;
import org.elasticsearch.client.Requests;
@ -562,8 +563,10 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
routing(metaData.resolveIndexRouting(routing, index));
// resolve timestamp if provided externally
if (timestamp != null) {
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp,
mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER);
mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER,
version);
}
// extract values if needed
if (mappingMd != null) {
@ -586,7 +589,8 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp();
if (timestamp != null) {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter(), version);
}
}
} catch (MapperParsingException e) {
@ -638,7 +642,8 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis());
} else {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
Version version = Version.indexCreated(metaData.getIndices().get(concreteIndex).settings());
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter(), version);
}
}
}

17
core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.cluster.AbstractDiffable;
import org.elasticsearch.common.Nullable;
@ -160,10 +161,22 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
public static class Timestamp {
public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter) throws TimestampParsingException {
private static final FormatDateTimeFormatter EPOCH_MILLIS_PARSER = Joda.forPattern("epoch_millis");
public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter,
Version version) throws TimestampParsingException {
try {
// no need for unix timestamp parsing in 2.x
FormatDateTimeFormatter formatter = version.onOrAfter(Version.V_2_0_0) ? dateTimeFormatter : EPOCH_MILLIS_PARSER;
return Long.toString(formatter.parser().parseMillis(timestampAsString));
} catch (RuntimeException e) {
if (version.before(Version.V_2_0_0)) {
try {
return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
} catch (RuntimeException e) {
} catch (RuntimeException e1) {
throw new TimestampParsingException(timestampAsString, e1);
}
}
throw new TimestampParsingException(timestampAsString, e);
}
}

30
core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.ToStringUtils;
import org.elasticsearch.Version;
import org.elasticsearch.action.fieldstats.FieldStats;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
@ -46,18 +47,17 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.NumericDateAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.core.LongFieldMapper.CustomLongNumericField;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.internal.SearchContext;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
@ -70,7 +70,7 @@ public class DateFieldMapper extends NumberFieldMapper {
public static final String CONTENT_TYPE = "date";
public static class Defaults extends NumberFieldMapper.Defaults {
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime", Locale.ROOT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime||epoch_millis", Locale.ROOT);
public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
public static final DateFieldType FIELD_TYPE = new DateFieldType();
@ -126,6 +126,14 @@ public class DateFieldMapper extends NumberFieldMapper {
protected void setupFieldType(BuilderContext context) {
FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
// TODO MOVE ME OUTSIDE OF THIS SPACE?
if (Version.indexCreated(context.indexSettings()).before(Version.V_2_0_0)) {
boolean includesEpochFormatter = dateTimeFormatter.format().contains("epoch_");
if (!includesEpochFormatter) {
String format = fieldType().timeUnit().equals(TimeUnit.SECONDS) ? "epoch_second" : "epoch_millis";
fieldType().setDateTimeFormatter(Joda.forPattern(format + "||" + dateTimeFormatter.format()));
}
}
if (!locale.equals(dateTimeFormatter.locale())) {
fieldType().setDateTimeFormatter(new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale));
}
@ -308,15 +316,7 @@ public class DateFieldMapper extends NumberFieldMapper {
}
protected long parseStringValue(String value) {
try {
return dateTimeFormatter().parser().parseMillis(value);
} catch (RuntimeException e) {
try {
return timeUnit().toMillis(Long.parseLong(value));
} catch (NumberFormatException e1) {
throw new MapperParsingException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter().format() + "], and timestamp number with locale [" + dateTimeFormatter().locale() + "]", e);
}
}
}
@Override

3
core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
View File

@ -147,6 +147,9 @@ public class RootObjectMapper extends ObjectMapper {
List<FormatDateTimeFormatter> dateTimeFormatters = newArrayList();
if (fieldNode instanceof List) {
for (Object node1 : (List) fieldNode) {
if (node1.toString().startsWith("epoch_")) {
throw new MapperParsingException("Epoch ["+ node1.toString() +"] is not supported as dynamic date format");
}
dateTimeFormatters.add(parseDateTimeFormatter(node1));
}
} else if ("none".equals(fieldNode.toString())) {

11
core/src/test/java/org/elasticsearch/count/simple/SimpleCountTests.java
View File

@ -181,7 +181,7 @@ public class SimpleCountTests extends ElasticsearchIntegrationTest {
}
@Test
public void testThatNonEpochDatesCanBeSearch() throws Exception {
public void testThatNonEpochDatesCanBeSearched() throws Exception {
assertAcked(prepareCreate("test")
.addMapping("type1",
jsonBuilder().startObject().startObject("type1")
@ -201,16 +201,9 @@ public class SimpleCountTests extends ElasticsearchIntegrationTest {
.endObject();
assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true));
// this is a timestamp in 2015 and should not be returned in counting when filtering by year
document = jsonBuilder()
.startObject()
.field("date_field", "1433236702")
.endObject();
assertThat(client().prepareIndex("test", "type1").setSource(document).get().isCreated(), is(true));
refresh();
assertHitCount(client().prepareCount("test").get(), 3);
assertHitCount(client().prepareCount("test").get(), 2);
CountResponse countResponse = client().prepareCount("test").setQuery(QueryBuilders.rangeQuery("date_field").from("2015010100").to("2015123123")).get();
assertHitCount(countResponse, 1);

205
core/src/test/java/org/elasticsearch/index/mapper/date/DateBackwardsCompatibilityTests.java Normal file
View File

@ -0,0 +1,205 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.mapper.date;
import com.google.common.collect.Lists;
import org.elasticsearch.Version;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.junit.Before;
import java.util.List;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
import static org.hamcrest.Matchers.is;
/**
* Test class to check for all the conditions defined in
* https://github.com/elastic/elasticsearch/issues/10971
*/
public class DateBackwardsCompatibilityTests extends ElasticsearchSingleNodeTest {
private String index = "testindex";
private String type = "testtype";
private Version randomVersionBelow2x;
@Before
public void setup() throws Exception {
randomVersionBelow2x = randomVersionBetween(getRandom(), Version.V_0_90_0, Version.V_1_6_1);
}
public void testThatPre2xIndicesNumbersAreTreatedAsEpochs() throws Exception {
index = createPre2xIndexAndMapping();
long dateInMillis = 1435073872l * 1000; // Tue Jun 23 17:37:52 CEST 2015
XContentBuilder document = jsonBuilder().startObject().field("date_field", dateInMillis).endObject();
index(document);
// search for date in time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertHitCount(response, 1);
}
public void testThatPre2xFailedStringParsingLeadsToEpochParsing() throws Exception {
index = createPre2xIndexAndMapping();
long dateInMillis = 1435073872l * 1000; // Tue Jun 23 17:37:52 CEST 2015
String date = String.valueOf(dateInMillis);
XContentBuilder document = jsonBuilder().startObject().field("date_field", date).endObject();
index(document);
// search for date in time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertHitCount(response, 1);
}
public void testThatPre2xSupportsUnixTimestampsInAnyDateFormat() throws Exception {
long dateInMillis = 1435073872l * 1000; // Tue Jun 23 17:37:52 CEST 2015
List<String> dateFormats = Lists.newArrayList("dateOptionalTime", "weekDate", "tTime", "ordinalDate", "hourMinuteSecond", "hourMinute");
for (String format : dateFormats) {
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format", format).endObject()
.endObject().endObject();
index = createIndex(randomVersionBelow2x, mapping);
XContentBuilder document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", String.valueOf(dateInMillis))
.endObject();
index(document);
// indexing as regular timestamp should work as well
document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", dateInMillis)
.endObject();
index(document);
client().admin().indices().prepareDelete(index).get();
}
}
public void testThatPre2xIndicesNumbersAreTreatedAsTimestamps() throws Exception {
// looks like a unix time stamp but is meant as 2016-06-23T01:00:00.000 - see the specified date format
long date = 2015062301000l;
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format","yyyyMMddHHSSS").endObject()
.endObject().endObject();
index = createIndex(randomVersionBelow2x, mapping);
XContentBuilder document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", randomBoolean() ? String.valueOf(date) : date)
.endObject();
index(document);
// no results in expected time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24").format("dateOptionalTime");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertNoSearchHits(response);
// result in unix timestamp range
QueryBuilder timestampQuery = QueryBuilders.rangeQuery("date_field").from(2015062300000L).to(2015062302000L);
assertHitCount(client().prepareSearch(index).setQuery(timestampQuery).get(), 1);
// result should also work with regular specified dates
QueryBuilder regularTimeQuery = QueryBuilders.rangeQuery("date_field").from("2033-11-08").to("2033-11-09").format("dateOptionalTime");
assertHitCount(client().prepareSearch(index).setQuery(regularTimeQuery).get(), 1);
}
public void testThatPost2xIndicesNumbersAreTreatedAsStrings() throws Exception {
// looks like a unix time stamp but is meant as 2016-06-23T01:00:00.000 - see the specified date format
long date = 2015062301000l;
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format","yyyyMMddHHSSS").endObject()
.endObject().endObject();
index = createIndex(Version.CURRENT, mapping);
XContentBuilder document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", String.valueOf(date))
.endObject();
index(document);
document = XContentFactory.jsonBuilder()
.startObject()
.field("date_field", date)
.endObject();
index(document);
// search for date in time range
QueryBuilder query = QueryBuilders.rangeQuery("date_field").from("2015-06-23").to("2015-06-24").format("dateOptionalTime");
SearchResponse response = client().prepareSearch(index).setQuery(query).get();
assertHitCount(response, 2);
}
public void testDynamicDateDetectionIn2xDoesNotSupportEpochs() throws Exception {
try {
XContentBuilder mapping = jsonBuilder().startObject()
.startArray("dynamic_date_formats").value("dateOptionalTime").value("epoch_seconds").endArray()
.endObject();
createIndex(Version.CURRENT, mapping);
fail("Expected a MapperParsingException, but did not happen");
} catch (MapperParsingException e) {
assertThat(e.getMessage(), is("mapping [" + type + "]"));
}
}
private String createPre2xIndexAndMapping() throws Exception {
return createIndexAndMapping(randomVersionBelow2x);
}
private String createIndexAndMapping(Version version) throws Exception {
XContentBuilder mapping = jsonBuilder().startObject().startObject("properties")
.startObject("date_field").field("type", "date").field("format", "dateOptionalTime").endObject()
.endObject().endObject();
return createIndex(version, mapping);
}
private String createIndex(Version version, XContentBuilder mapping) {
Settings settings = settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
createIndex(index, settings, type, mapping);
ensureGreen(index);
return index;
}
private void index(XContentBuilder document) {
IndexResponse indexResponse = client().prepareIndex(index, type).setSource(document).setRefresh(true).get();
assertThat(indexResponse.isCreated(), is(true));
}
}

39
core/src/test/java/org/elasticsearch/index/mapper/date/SimpleDateMappingTests.java
View File

@ -25,7 +25,9 @@ import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.util.Constants;
import org.elasticsearch.Version;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.LocaleUtils;
@ -53,6 +55,7 @@ import java.util.*;
import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.index.mapper.string.SimpleStringMappingTests.docValuesType;
import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
import static org.hamcrest.Matchers.*;
public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
@ -147,8 +150,17 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
int i = 0;
private DocumentMapper mapper(String type, String mapping) throws IOException {
return mapper(type, mapping, Version.CURRENT);
}
private DocumentMapper mapper(String type, String mapping, Version version) throws IOException {
final String indexName = "test-" + (i++);
IndexService index = createIndex(indexName);
IndexService index;
if (version.equals(Version.CURRENT)) {
index = createIndex(indexName);
} else {
index = createIndex(indexName, settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build());
}
client().admin().indices().preparePutMapping(indexName).setType(type).setSource(mapping).get();
return index.mapperService().documentMapper(type);
}
@ -181,8 +193,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject().endObject().string();
DocumentMapper defaultMapper = mapper("type", mapping);
long value = System.currentTimeMillis();
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("date_field", value)
@ -290,7 +302,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class));
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), is("failed to parse [field2]"));
}
// Verify that the default is false
@ -301,7 +314,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class));
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), is("failed to parse [field3]"));
}
// Unless the global ignore_malformed option is set to true
@ -322,7 +336,8 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject()
.bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(MapperParsingException.class));
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), is("failed to parse [field2]"));
}
}
@ -399,12 +414,12 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
throw new AssertionError("missing");
}
public void testNumericResolution() throws Exception {
public void testNumericResolutionBackwardsCompat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("date_field").field("type", "date").field("format", "date_time").field("numeric_resolution", "seconds").endObject().endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = mapper("type", mapping);
DocumentMapper defaultMapper = mapper("type", mapping, Version.V_0_90_0);
// provided as an int
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
@ -429,6 +444,16 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
.endObject()
.bytes());
assertThat(getDateAsMillis(doc.rootDoc(), "date_field"), equalTo(44000L));
// expected to fail due to field epoch date formatters not being set
DocumentMapper currentMapper = mapper("type", mapping);
try {
currentMapper.parse("type", "2", XContentFactory.jsonBuilder()
.startObject()
.field("date_field", randomBoolean() ? "43" : 43)
.endObject()
.bytes());
} catch (MapperParsingException e) {}
}
public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception {

67
core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
View File

@ -38,12 +38,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MergeResult;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.junit.Test;
@ -57,7 +52,9 @@ import java.util.Map;
import static org.elasticsearch.Version.V_1_5_0;
import static org.elasticsearch.Version.V_2_0_0;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.test.VersionUtils.randomVersion;
import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -249,15 +246,15 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping.string());
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), notNullValue());
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"))));
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"), Version.CURRENT)));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
@ -274,15 +271,15 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.field("foo", "bar")
.endObject();
MetaData metaData = MetaData.builder().build();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), notNullValue());
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"))));
assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"), Version.CURRENT)));
}
@Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
@ -751,11 +748,12 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.startObject("_timestamp").field("enabled", true).field("path", "custom_timestamp").endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("custom_timestamp", 1).endObject();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(MetaData.builder().build(), mappingMetaData, true, "test");
request.process(metaData, mappingMetaData, true, "test");
assertEquals(request.timestamp(), "1");
}
@ -766,14 +764,15 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.endObject().endObject().string();
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
DocumentMapper docMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("_timestamp", 2000000).endObject();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(MetaData.builder().build(), mappingMetaData, true, "test");
request.process(metaData, mappingMetaData, true, "test");
// _timestamp in a document never worked, so backcompat is ignoring the field
assertEquals(MappingMetaData.Timestamp.parseStringTimestamp("1970", Joda.forPattern("YYYY")), request.timestamp());
assertEquals(MappingMetaData.Timestamp.parseStringTimestamp("1970", Joda.forPattern("YYYY"), Version.V_1_4_2), request.timestamp());
assertNull(docMapper.parse("type", "1", doc.bytes()).rootDoc().get("_timestamp"));
}
@ -782,12 +781,52 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
.startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject();
IndexRequest request = new IndexRequest("test", "type", "1").source(doc).timestamp("2015060210");
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
request.process(MetaData.builder().build(), mappingMetaData, true, "test");
request.process(metaData, mappingMetaData, true, "test");
assertThat(request.timestamp(), is("1433239200000"));
}
public void testThatIndicesBefore2xMustSupportUnixTimestampsInAnyDateFormat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true).field("format", "dateOptionalTime").endObject()
.endObject().endObject().string();
BytesReference source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
//
// test with older versions
Settings oldSettings = settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, randomVersionBetween(random(), Version.V_0_90_0, Version.V_1_6_0)).build();
DocumentMapper docMapper = createIndex("old-index", oldSettings).mapperService().documentMapperParser().parse(mapping);
MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
// both index request are successfully processed
IndexRequest oldIndexDateIndexRequest = new IndexRequest("old-index", "type", "1").source(source).timestamp("1970-01-01");
oldIndexDateIndexRequest.process(metaData, new MappingMetaData(docMapper), true, "old-index");
IndexRequest oldIndexTimestampIndexRequest = new IndexRequest("old-index", "type", "1").source(source).timestamp("1234567890");
oldIndexTimestampIndexRequest.process(metaData, new MappingMetaData(docMapper), true, "old-index");
//
// test with 2.x
DocumentMapper currentMapper = createIndex("new-index").mapperService().documentMapperParser().parse(mapping);
MetaData newMetaData = client().admin().cluster().prepareState().get().getState().getMetaData();
// this works with 2.x
IndexRequest request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1970-01-01");
request.process(newMetaData, new MappingMetaData(currentMapper), true, "new-index");
// this fails with 2.x
request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1234567890");
try {
request.process(newMetaData, new MappingMetaData(currentMapper), true, "new-index");
} catch (Exception e) {
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(e.getMessage(), containsString("failed to parse timestamp [1234567890]"));
}
}
}

2
core/src/test/java/org/elasticsearch/percolator/PercolatorTests.java
View File

@ -1813,7 +1813,7 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
@Test
public void testPercolatorQueryWithNowRange() throws Exception {
client().admin().indices().prepareCreate("test")
.addMapping("my-type", "timestamp", "type=date")
.addMapping("my-type", "timestamp", "type=date,format=epoch_millis")
.get();
ensureGreen();

2
core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreTests.java
View File

@ -449,7 +449,7 @@ public class DecayFunctionScoreTests extends ElasticsearchIntegrationTest {
assertAcked(prepareCreate("test").addMapping(
"type1",
jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("test").field("type", "string")
.endObject().startObject("num1").field("type", "date").endObject().endObject().endObject().endObject()));
.endObject().startObject("num1").field("type", "date").field("format", "epoch_millis").endObject().endObject().endObject().endObject()));
ensureYellow();
client().index(
indexRequest("test").type("type1").id("1")

854
dev-tools/build_release.py
View File

File diff suppressed because it is too large Load Diff

2
docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
View File

@ -58,7 +58,7 @@ Response:
[[date-format-pattern]]
==== Date Format/Pattern
NOTE: this information was copied from http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html[JodaDate]
NOTE: this information was copied from http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[JodaDate]
All ASCII letters are reserved as format pattern letters, which are defined as follows:

6
docs/reference/mapping/date-format.asciidoc
View File

@ -8,9 +8,9 @@ specifying `dynamic_date_formats` in the `root object` mapping (which will
be used unless explicitly overridden by a `date` type). There are built in
formats supported, as well as complete custom one.
The parsing of dates uses http://joda-time.sourceforge.net/[Joda]. The
The parsing of dates uses http://www.joda.org/joda-time/[Joda]. The
default date parsing used if no format is specified is
http://joda-time.sourceforge.net/api-release/org/joda/time/format/ISODateTimeFormat.html#dateOptionalTimeParser()[ISODateTimeFormat.dateOptionalTimeParser].
http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateOptionalTimeParser--[ISODateTimeFormat.dateOptionalTimeParser].
An extension to the format allow to define several formats using `||`
separator. This allows to define less strict formats that can be used,
@ -215,4 +215,4 @@ date formatter in that case.
=== Custom Format
Allows for a completely customizable date format explained
http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html[here].
http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].

3
docs/reference/mapping/types/core-types.asciidoc
View File

@ -378,9 +378,6 @@ defaults to `true` or to the parent `object` type setting.
|`ignore_malformed` |Ignored a malformed number. Defaults to `false`.
|`numeric_resolution` |The unit to use when passed in a numeric values. Possible
values include `seconds` and `milliseconds` (default).
|=======================================================================
[float]

3
docs/reference/migration/migrate_2_0.asciidoc
View File

@ -312,6 +312,9 @@ date.
This is not supported anymore. If you want to store unix timestamps, you need to specify
the appropriate formats in the mapping, namely `epoch_second` or `epoch_millis`.
In addition the `numeric_resolution` mapping parameter is ignored. Use the
`epoch_second` and `epoch_millis` date formats instead.
==== Source field limitations
The `_source` field could previously be disabled dynamically. Since this field
is a critical piece of many features like the Update API, it is no longer

2
docs/reference/query-dsl/query-string-query.asciidoc
View File

@ -80,7 +80,7 @@ providing text to a numeric field) to be ignored.
Defaults to `ROOT`.
|`time_zone` | Time Zone to be applied to any range query related to dates. See also
http://joda-time.sourceforge.net/api-release/org/joda/time/DateTimeZone.html[JODA timezone].
http://www.joda.org/joda-time/apidocs/org/joda/time/DateTimeZone.html[JODA timezone].
|=======================================================================
When a multi term query is being generated, one can control how it gets