honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 10:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve DateFieldMapper ignore_malformed handling (#50090)

Browse Source 
A recent change around date parsing (#46675) made it stricter, so we should now also
catch DateTimeExceptions in DateFieldMapper and ignore those when the
`ignore_malformed` option is set.

Closes #50081
This commit is contained in:
Christoph Büscher 2019-12-13 09:59:45 +01:00
parent e6cbcf7f7c
commit c0216f9a06
2 changed files with 20 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
View File

@ -54,6 +54,7 @@ import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.DocValueFormat;
import java.io.IOException;
import java.time.DateTimeException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
@ -77,19 +78,23 @@ public final class DateFieldMapper extends FieldMapper {
public enum Resolution {
MILLISECONDS(CONTENT_TYPE, NumericType.DATE) {
@Override
public long convert(Instant instant) {
return instant.toEpochMilli();
}
@Override
public Instant toInstant(long value) {
return Instant.ofEpochMilli(value);
}
},
NANOSECONDS("date_nanos", NumericType.DATE_NANOSECONDS) {
@Override
public long convert(Instant instant) {
return toLong(instant);
}
@Override
public Instant toInstant(long value) {
return DateUtils.toInstant(value);
}
@ -274,7 +279,9 @@ public final class DateFieldMapper extends FieldMapper {
@Override
public boolean equals(Object o) {
if (!super.equals(o)) return false;
if (!super.equals(o)) {
return false;
}
DateFieldType that = (DateFieldType) o;
return Objects.equals(dateTimeFormatter, that.dateTimeFormatter) && Objects.equals(resolution, that.resolution);
}
@ -536,7 +543,7 @@ public final class DateFieldMapper extends FieldMapper {
long timestamp;
try {
timestamp = fieldType().parse(dateAsString);
} catch (IllegalArgumentException | ElasticsearchParseException e) {
} catch (IllegalArgumentException | ElasticsearchParseException | DateTimeException e) {
if (ignoreMalformed.value()) {
context.addIgnoredField(fieldType.name());
return;

16
server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
View File

@ -159,7 +159,14 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
assertEquals(1457654400000L, storedField.numericValue().longValue());
}
public void testIgnoreMalformed() throws Exception {
public void testIgnoreMalformed() throws IOException {
testIgnoreMalfomedForValue("2016-03-99",
"failed to parse date field [2016-03-99] with format [strict_date_optional_time||epoch_millis]");
testIgnoreMalfomedForValue("-2147483648",
"Invalid value for Year (valid values -999999999 - 999999999): -2147483648");
}
private void testIgnoreMalfomedForValue(String value, String expectedException) throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "date").endObject().endObject()
.endObject().endObject());
@ -171,12 +178,11 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
ThrowingRunnable runnable = () -> mapper.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "2016-03-99")
.field("field", value)
.endObject()),
XContentType.JSON));
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
assertThat(e.getCause().getMessage(),
containsString("failed to parse date field [2016-03-99] with format [strict_date_optional_time||epoch_millis]"));
assertThat(e.getCause().getMessage(), containsString(expectedException));
mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", "date")
@ -188,7 +194,7 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
ParsedDocument doc = mapper2.parse(new SourceToParse("test", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", ":1")
.field("field", value)
.endObject()),
XContentType.JSON));