mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-25 01:19:02 +00:00
Add BWC compatible processing to ingest date processors (#37407)
The ingest date processor is currently only able to parse joda formats. However it is not using the existing elasticsearch classes but access joda directly. This means that our existing BWC layer does not notify the user about deprecated formats. This commit switches to use the exising Elasticsearch Joda methods to acquire a date format, that includes the BWC check and the ability to parse java 8 dates. The date parsing in ingest has also another extra feature, that the fallback year, when a date format without a year is used, is the current year, and not 1970 like usual. This is currently not properly supported in the DateFormatter class. As this is the only case for this feature and java time can take care of this using the toZonedDateTime() method, a workaround just for the joda time parser has been created, that can be removed soon again from 7.0.
This commit is contained in:
Alexander Reelsen
GitHub
parent
787acb14b9
commit
9e350d027e
@ -19,12 +19,19 @@
|
||||
|
||||
package org.elasticsearch.ingest.common;
|
||||
|
||||
import org.elasticsearch.common.joda.Joda;
|
||||
import org.elasticsearch.common.time.DateFormatter;
|
||||
import org.elasticsearch.common.time.DateFormatters;
|
||||
import org.elasticsearch.common.time.DateUtils;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
import org.joda.time.format.ISODateTimeFormat;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.temporal.TemporalAccessor;
|
||||
import java.util.Locale;
|
||||
import java.util.function.Function;
|
||||
|
||||
@ -63,11 +70,33 @@ enum DateFormat {
|
||||
return ((base * 1000) - 10000) + (rest/1000000);
|
||||
}
|
||||
},
|
||||
Joda {
|
||||
Java {
|
||||
@Override
|
||||
Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
|
||||
DateTimeFormatter parser = DateTimeFormat.forPattern(format).withZone(timezone).withLocale(locale);
|
||||
return text -> parser.withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear()).parseDateTime(text);
|
||||
// in case you are wondering why we do not call 'DateFormatter.forPattern(format)' for all cases here, but only for the
|
||||
// non java time case:
|
||||
// When the joda date formatter parses a date then a year is always set, so that no fallback can be used, like
|
||||
// done in the JodaDateFormatter.withYear() code below
|
||||
// This means that we leave the existing parsing logic in place, but will fall back to the new java date parsing logic, if an
|
||||
// "8" is prepended to the date format string
|
||||
int year = LocalDate.now(ZoneOffset.UTC).getYear();
|
||||
if (format.startsWith("8")) {
|
||||
DateFormatter formatter = DateFormatter.forPattern(format)
|
||||
.withLocale(locale)
|
||||
.withZone(DateUtils.dateTimeZoneToZoneId(timezone));
|
||||
return text -> {
|
||||
ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year);
|
||||
TemporalAccessor accessor = formatter.parse(text);
|
||||
long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli();
|
||||
return new DateTime(millis, timezone);
|
||||
};
|
||||
} else {
|
||||
DateFormatter formatter = Joda.forPattern(format)
|
||||
.withYear(year)
|
||||
.withZone(DateUtils.dateTimeZoneToZoneId(timezone))
|
||||
.withLocale(locale);
|
||||
return text -> new DateTime(formatter.parseMillis(text), timezone);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@ -84,7 +113,7 @@ enum DateFormat {
|
||||
case "TAI64N":
|
||||
return Tai64n;
|
||||
default:
|
||||
return Joda;
|
||||
return Java;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -34,7 +34,7 @@ import static org.hamcrest.core.IsEqual.equalTo;
|
||||
public class DateFormatTests extends ESTestCase {
|
||||
|
||||
public void testParseJoda() {
|
||||
Function<String, DateTime> jodaFunction = DateFormat.Joda.getFunction("MMM dd HH:mm:ss Z",
|
||||
Function<String, DateTime> jodaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z",
|
||||
DateTimeZone.forOffsetHours(-8), Locale.ENGLISH);
|
||||
assertThat(Instant.ofEpochMilli(jodaFunction.apply("Nov 24 01:29:01 -0800").getMillis())
|
||||
.atZone(ZoneId.of("GMT-8"))
|
||||
@ -78,13 +78,13 @@ public class DateFormatTests extends ESTestCase {
|
||||
|
||||
public void testFromString() {
|
||||
assertThat(DateFormat.fromString("UNIX_MS"), equalTo(DateFormat.UnixMs));
|
||||
assertThat(DateFormat.fromString("unix_ms"), equalTo(DateFormat.Joda));
|
||||
assertThat(DateFormat.fromString("unix_ms"), equalTo(DateFormat.Java));
|
||||
assertThat(DateFormat.fromString("UNIX"), equalTo(DateFormat.Unix));
|
||||
assertThat(DateFormat.fromString("unix"), equalTo(DateFormat.Joda));
|
||||
assertThat(DateFormat.fromString("unix"), equalTo(DateFormat.Java));
|
||||
assertThat(DateFormat.fromString("ISO8601"), equalTo(DateFormat.Iso8601));
|
||||
assertThat(DateFormat.fromString("iso8601"), equalTo(DateFormat.Joda));
|
||||
assertThat(DateFormat.fromString("iso8601"), equalTo(DateFormat.Java));
|
||||
assertThat(DateFormat.fromString("TAI64N"), equalTo(DateFormat.Tai64n));
|
||||
assertThat(DateFormat.fromString("tai64n"), equalTo(DateFormat.Joda));
|
||||
assertThat(DateFormat.fromString("prefix-" + randomAlphaOfLengthBetween(1, 10)), equalTo(DateFormat.Joda));
|
||||
assertThat(DateFormat.fromString("tai64n"), equalTo(DateFormat.Java));
|
||||
assertThat(DateFormat.fromString("prefix-" + randomAlphaOfLengthBetween(1, 10)), equalTo(DateFormat.Java));
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,7 +35,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
|
||||
public class DateIndexNameProcessorTests extends ESTestCase {
|
||||
|
||||
public void testJodaPattern() throws Exception {
|
||||
Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
|
||||
Function<String, DateTime> function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
|
||||
DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
|
||||
DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
|
||||
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
|
||||
|
||||
@ -109,7 +109,7 @@ public class DateProcessorTests extends ESTestCase {
|
||||
fail("date processor execution should have failed");
|
||||
} catch(IllegalArgumentException e) {
|
||||
assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
|
||||
assertThat(e.getCause().getMessage(), equalTo("Illegal pattern component: i"));
|
||||
assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Illegal pattern component: i"));
|
||||
}
|
||||
}
|
||||
|
||||
@ -127,9 +127,10 @@ public class DateProcessorTests extends ESTestCase {
|
||||
}
|
||||
|
||||
public void testJodaPatternDefaultYear() {
|
||||
String format = randomFrom("dd/MM", "8dd/MM");
|
||||
DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
|
||||
templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
|
||||
"date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
|
||||
"date_as_string", Collections.singletonList(format), "date_as_date");
|
||||
Map<String, Object> document = new HashMap<>();
|
||||
document.put("date_as_string", "12/06");
|
||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
||||
|
||||
@ -1,5 +1,10 @@
|
||||
---
|
||||
"Test logging":
|
||||
- skip:
|
||||
version: " - 6.9.99"
|
||||
reason: pre-7.0.0 will send no warnings
|
||||
features: "warnings"
|
||||
|
||||
- do:
|
||||
ingest.put_pipeline:
|
||||
id: "_id"
|
||||
@ -41,6 +46,8 @@
|
||||
- match: { acknowledged: true }
|
||||
|
||||
- do:
|
||||
warnings:
|
||||
- "Use of 'Y' (year-of-era) will change to 'y' in the next major version of Elasticsearch. Prefix your date format with '8' to use the new specifier."
|
||||
index:
|
||||
index: test
|
||||
type: test
|
||||
|
||||
@ -107,8 +107,8 @@ public class Joda {
|
||||
// in this case, we have a separate parser and printer since the dataOptionalTimeParser can't print
|
||||
// this sucks we should use the root local by default and not be dependent on the node
|
||||
return new JodaDateFormatter(input,
|
||||
ISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC),
|
||||
ISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC));
|
||||
ISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970),
|
||||
ISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970));
|
||||
} else if ("dateTime".equals(input) || "date_time".equals(input)) {
|
||||
formatter = ISODateTimeFormat.dateTime();
|
||||
} else if ("dateTimeNoMillis".equals(input) || "date_time_no_millis".equals(input)) {
|
||||
@ -184,8 +184,9 @@ public class Joda {
|
||||
// in this case, we have a separate parser and printer since the dataOptionalTimeParser can't print
|
||||
// this sucks we should use the root local by default and not be dependent on the node
|
||||
return new JodaDateFormatter(input,
|
||||
StrictISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC),
|
||||
StrictISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC));
|
||||
StrictISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC)
|
||||
.withDefaultYear(1970),
|
||||
StrictISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970));
|
||||
} else if ("strictDateTime".equals(input) || "strict_date_time".equals(input)) {
|
||||
formatter = StrictISODateTimeFormat.dateTime();
|
||||
} else if ("strictDateTimeNoMillis".equals(input) || "strict_date_time_no_millis".equals(input)) {
|
||||
@ -262,7 +263,7 @@ public class Joda {
|
||||
}
|
||||
}
|
||||
|
||||
formatter = formatter.withLocale(Locale.ROOT).withZone(DateTimeZone.UTC);
|
||||
formatter = formatter.withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970);
|
||||
return new JodaDateFormatter(input, formatter, formatter);
|
||||
}
|
||||
|
||||
@ -311,7 +312,7 @@ public class Joda {
|
||||
DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder().append(longFormatter.withZone(DateTimeZone.UTC).getPrinter(),
|
||||
new DateTimeParser[]{longFormatter.getParser(), shortFormatter.getParser(), new EpochTimeParser(true)});
|
||||
|
||||
DateTimeFormatter formatter = builder.toFormatter().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC);
|
||||
DateTimeFormatter formatter = builder.toFormatter().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970);
|
||||
return new JodaDateFormatter("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis", formatter, formatter);
|
||||
}
|
||||
|
||||
|
||||
@ -39,10 +39,10 @@ public class JodaDateFormatter implements DateFormatter {
|
||||
final DateTimeFormatter parser;
|
||||
final DateTimeFormatter printer;
|
||||
|
||||
public JodaDateFormatter(String pattern, DateTimeFormatter parser, DateTimeFormatter printer) {
|
||||
JodaDateFormatter(String pattern, DateTimeFormatter parser, DateTimeFormatter printer) {
|
||||
this.pattern = pattern;
|
||||
this.printer = printer.withDefaultYear(1970);
|
||||
this.parser = parser.withDefaultYear(1970);
|
||||
this.printer = printer;
|
||||
this.parser = parser;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -62,6 +62,9 @@ public class JodaDateFormatter implements DateFormatter {
|
||||
@Override
|
||||
public DateFormatter withZone(ZoneId zoneId) {
|
||||
DateTimeZone timeZone = DateUtils.zoneIdToDateTimeZone(zoneId);
|
||||
if (parser.getZone().equals(timeZone)) {
|
||||
return this;
|
||||
}
|
||||
DateTimeFormatter parser = this.parser.withZone(timeZone);
|
||||
DateTimeFormatter printer = this.printer.withZone(timeZone);
|
||||
return new JodaDateFormatter(pattern, parser, printer);
|
||||
@ -69,6 +72,9 @@ public class JodaDateFormatter implements DateFormatter {
|
||||
|
||||
@Override
|
||||
public DateFormatter withLocale(Locale locale) {
|
||||
if (parser.getLocale().equals(locale)) {
|
||||
return this;
|
||||
}
|
||||
DateTimeFormatter parser = this.parser.withLocale(locale);
|
||||
DateTimeFormatter printer = this.printer.withLocale(locale);
|
||||
return new JodaDateFormatter(pattern, parser, printer);
|
||||
@ -89,6 +95,13 @@ public class JodaDateFormatter implements DateFormatter {
|
||||
return printer.print(millis);
|
||||
}
|
||||
|
||||
public JodaDateFormatter withYear(int year) {
|
||||
if (parser.getDefaultYear() == year) {
|
||||
return this;
|
||||
}
|
||||
return new JodaDateFormatter(pattern, parser.withDefaultYear(year), printer.withDefaultYear(year));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String pattern() {
|
||||
return pattern;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user