honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch default time format for ingest from Joda to Java for v7 (#37934) 

Date formats with and without the "8" prefix are now all treated
as Java time formats, so that ingest does the same as mappings
in this respect.
This commit is contained in:
David Roberts 2019-01-30 16:26:28 +00:00 committed by GitHub
parent ecbaa38864
commit 2f7776c8b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 31 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.time.DateUtils;
@ -73,30 +72,22 @@ enum DateFormat {
Java {
@Override
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
// in case you are wondering why we do not call 'DateFormatter.forPattern(format)' for all cases here, but only for the
// non java time case:
// When the joda date formatter parses a date then a year is always set, so that no fallback can be used, like
// done in the JodaDateFormatter.withYear() code below
// This means that we leave the existing parsing logic in place, but will fall back to the new java date parsing logic, if an
// "8" is prepended to the date format string
int year = LocalDate.now(ZoneOffset.UTC).getYear();
// support the 6.x BWC compatible way of parsing java 8 dates
if (format.startsWith("8")) {
DateFormatter formatter = DateFormatter.forPattern(format)
.withLocale(locale)
.withZone(DateUtils.dateTimeZoneToZoneId(timezone));
return text -> {
ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year);
TemporalAccessor accessor = formatter.parse(text);
long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli();
return new DateTime(millis, timezone);
};
} else {
DateFormatter formatter = Joda.forPattern(format)
.withYear(year)
.withZone(DateUtils.dateTimeZoneToZoneId(timezone))
.withLocale(locale);
return text -> new DateTime(formatter.parseMillis(text), timezone);
format = format.substring(1);
}
int year = LocalDate.now(ZoneOffset.UTC).getYear();
DateFormatter formatter = DateFormatter.forPattern(format)
.withLocale(locale)
.withZone(DateUtils.dateTimeZoneToZoneId(timezone));
return text -> {
ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year);
TemporalAccessor accessor = formatter.parse(text);
long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli();
return new DateTime(millis, timezone);
};
}
};

2
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
View File

@ -157,7 +157,7 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
}
List<String> dateFormatStrings = ConfigurationUtils.readOptionalList(TYPE, tag, config, "date_formats");
if (dateFormatStrings == null) {
dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSXX");
}
List<Function<String, DateTime>> dateFormats = new ArrayList<>(dateFormatStrings.size());
for (String format : dateFormatStrings) {

6
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
View File

@ -34,8 +34,8 @@ import static org.hamcrest.CoreMatchers.equalTo;
public class DateIndexNameProcessorTests extends ESTestCase {
public void testJodaPattern() throws Exception {
Function<String, DateTime> function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
public void testJavaPattern() throws Exception {
Function<String, DateTime> function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSXX", DateTimeZone.UTC, Locale.ROOT);
DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
@ -82,7 +82,7 @@ public class DateIndexNameProcessorTests extends ESTestCase {
public void testTemplatedFields() throws Exception {
String indexNamePrefix = randomAlphaOfLength(10);
String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s");
String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy");
String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSXX", "yyyyMMdd", "MM/dd/yyyy");
String date = Integer.toString(randomInt());
Function<String, DateTime> dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);

21
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
View File

@ -49,10 +49,11 @@ public class DateProcessorTests extends ESTestCase {
String id = timezone.equals(ZoneOffset.UTC) ? "UTC" : timezone.getId();
return new TestTemplateService.MockTemplateScript.Factory(id);
}
public void testJodaPattern() {
public void testJavaPattern() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
"date_as_string", Collections.singletonList("yyyy dd MM hh:mm:ss"), "date_as_date");
"date_as_string", Collections.singletonList("yyyy dd MM HH:mm:ss"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "2010 12 06 11:05:15");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
@ -60,7 +61,7 @@ public class DateProcessorTests extends ESTestCase {
assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T11:05:15.000+02:00"));
}
public void testJodaPatternMultipleFormats() {
public void testJavaPatternMultipleFormats() {
List<String> matchFormats = new ArrayList<>();
matchFormats.add("yyyy dd MM");
matchFormats.add("dd/MM/yyyy");
@ -98,7 +99,7 @@ public class DateProcessorTests extends ESTestCase {
}
}
public void testInvalidJodaPattern() {
public void testInvalidJavaPattern() {
try {
DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),
templatize(ZoneOffset.UTC), templatize(randomLocale(random())),
@ -109,16 +110,14 @@ public class DateProcessorTests extends ESTestCase {
fail("date processor execution should have failed");
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Illegal pattern component: i"));
assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Unknown pattern letter: i"));
}
}
public void testJodaPatternLocale() {
//TODO investigate if this is a bug in Joda
assumeFalse("Can't run in a FIPS JVM, Joda parse date error", inFipsJvm());
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
public void testJavaPatternLocale() {
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ITALIAN),
"date_as_string", Collections.singletonList("yyyy dd MMM"), "date_as_date");
"date_as_string", Collections.singletonList("yyyy dd MMMM"), "date_as_date");
Map<String, Object> document = new HashMap<>();
document.put("date_as_string", "2010 12 giugno");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
@ -126,7 +125,7 @@ public class DateProcessorTests extends ESTestCase {
assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T00:00:00.000+02:00"));
}
public void testJodaPatternDefaultYear() {
public void testJavaPatternDefaultYear() {
String format = randomFrom("dd/MM", "8dd/MM");
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),

8
qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml
View File

@ -1,8 +1,8 @@
---
"Test logging":
"Test with date processor":
- skip:
version: " - 6.9.99"
reason: pre-7.0.0 will send no warnings
reason: pre-7.0.0 requires the 8 prefix for Java time formats, so would treat the format in this test as a Joda time format
features: "warnings"
- do:
@ -33,7 +33,7 @@
"date" : {
"field" : "timestamp",
"target_field" : "timestamp",
"formats" : ["dd/MMM/YYYY:HH:mm:ss Z"]
"formats" : ["dd/MMM/yyyy:HH:mm:ss xx"]
}
},
{
@ -46,8 +46,6 @@
- match: { acknowledged: true }
- do:
warnings:
- "Use of 'Y' (year-of-era) will change to 'y' in the next major version of Elasticsearch. Prefix your date format with '8' to use the new specifier."
index:
index: test
type: test