honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Enable ResolverStyle.STRICT for java formatters backport(#46675) (#47913) 

Joda was using ResolverStyle.STRICT when parsing. This means that date will be validated to be a correct year, year-of-month, day-of-month
However, we also want to make it works with Year-Of-Era as Joda used to, hence custom temporalquery.localdate in DateFormatters.from
Within DateFormatters we use the correct uuuu year instead of yyyy year of era

worth noting: if yyyy(without an era) is used in code, the parsing result will be a TemporalAccessor which will fail to be converted into LocalDate. We mostly use DateFormatters.from so this takes care of this. If possible the uuuu format should be used.
This commit is contained in:
Przemyslaw Gomulka 2019-10-11 21:19:56 +02:00 committed by GitHub
parent 627faf1850
commit 6ab58de7ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 562 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
View File

@ -98,9 +98,9 @@ enum DateFormat {
final DateFormatter formatter = dateFormatter; final DateFormatter formatter = dateFormatter;
return text -> { return text -> {
TemporalAccessor accessor = formatter.parse(text); TemporalAccessor accessor = formatter.parse(text);
// if there is no year, we fall back to the current one and // if there is no year nor year-of-era, we fall back to the current one and
// fill the rest of the date up with the parsed date // fill the rest of the date up with the parsed date
if (accessor.isSupported(ChronoField.YEAR) == false) { if (accessor.isSupported(ChronoField.YEAR) == false && accessor.isSupported(ChronoField.YEAR_OF_ERA) == false ) {
int year = LocalDate.now(ZoneOffset.UTC).getYear(); int year = LocalDate.now(ZoneOffset.UTC).getYear();
ZonedDateTime newTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); ZonedDateTime newTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year);
for (ChronoField field : FIELDS) { for (ChronoField field : FIELDS) {

10
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
View File

@ -44,7 +44,7 @@ public class DateFormatTests extends ESTestCase {
equalTo("11 24 01:29:01")); equalTo("11 24 01:29:01"));
} }
public void testParseJavaWithTimeZone() { public void testParseYearOfEraJavaWithTimeZone() {
Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZZ",
ZoneOffset.UTC, Locale.ROOT); ZoneOffset.UTC, Locale.ROOT);
ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100"); ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100");
@ -52,6 +52,14 @@ public class DateFormatTests extends ESTestCase {
assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z")); assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z"));
} }
public void testParseYearJavaWithTimeZone() {
Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction("uuuu-MM-dd'T'HH:mm:ss.SSSZZ",
ZoneOffset.UTC, Locale.ROOT);
ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100");
String expectedDateTime = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").withZone(ZoneOffset.UTC).format(datetime);
assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z"));
}
public void testParseJavaDefaultYear() { public void testParseJavaDefaultYear() {
String format = randomFrom("8dd/MM", "dd/MM"); String format = randomFrom("8dd/MM", "dd/MM");
ZoneId timezone = DateUtils.of("Europe/Amsterdam"); ZoneId timezone = DateUtils.of("Europe/Amsterdam");

770
server/src/main/java/org/elasticsearch/common/time/DateFormatters.java
View File

File diff suppressed because it is too large Load Diff

4
server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java
View File

@ -872,6 +872,10 @@ public class JavaJodaTimeDuellingTests extends ESTestCase {
assertSameDate("2018-10-10T10:11:12,123+05:30", format, jodaFormatter, javaFormatter); assertSameDate("2018-10-10T10:11:12,123+05:30", format, jodaFormatter, javaFormatter);
} }
public void testParsingLocalDateFromYearOfEra(){
//with strict resolving, YearOfEra expect an era, otherwise it won't resolve to a date
assertSameDate("2018363","yyyyDDD",Joda.forPattern("YYYYDDD"),DateFormatter.forPattern("uuuuDDD"));
}
public void testParsingMissingTimezone() { public void testParsingMissingTimezone() {
long millisJava = DateFormatter.forPattern("8yyyy-MM-dd HH:mm:ss").parseMillis("2018-02-18 17:47:17"); long millisJava = DateFormatter.forPattern("8yyyy-MM-dd HH:mm:ss").parseMillis("2018-02-18 17:47:17");
long millisJoda = DateFormatter.forPattern("yyyy-MM-dd HH:mm:ss").parseMillis("2018-02-18 17:47:17"); long millisJoda = DateFormatter.forPattern("yyyy-MM-dd HH:mm:ss").parseMillis("2018-02-18 17:47:17");

12
server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java
View File

@ -198,7 +198,7 @@ public class DateDerivativeIT extends ESIntegTestCase {
List<IndexRequestBuilder> builders = new ArrayList<>(); List<IndexRequestBuilder> builders = new ArrayList<>();
ZoneId timezone = ZoneId.of("CET"); ZoneId timezone = ZoneId.of("CET");
DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); DateFormatter formatter = DateFormatter.forPattern("uuuu-MM-dd'T'HH:mm:ss").withZone(timezone);
// epoch millis: 1332547200000 // epoch millis: 1332547200000
addNTimes(1, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-24T01:00:00")), builders); addNTimes(1, IDX_DST_START, DateFormatters.from(formatter.parse("2012-03-24T01:00:00")), builders);
// day with dst shift, only 23h long // day with dst shift, only 23h long
@ -223,7 +223,7 @@ public class DateDerivativeIT extends ESIntegTestCase {
List<? extends Bucket> buckets = deriv.getBuckets(); List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(4)); assertThat(buckets.size(), equalTo(4));
DateFormatter dateFormatter = DateFormatter.forPattern("yyyy-MM-dd"); DateFormatter dateFormatter = DateFormatter.forPattern("uuuu-MM-dd");
ZonedDateTime expectedKeyFirstBucket = ZonedDateTime expectedKeyFirstBucket =
LocalDate.from(dateFormatter.parse("2012-03-24")).atStartOfDay(timezone).withZoneSameInstant(ZoneOffset.UTC); LocalDate.from(dateFormatter.parse("2012-03-24")).atStartOfDay(timezone).withZoneSameInstant(ZoneOffset.UTC);
assertBucket(buckets.get(0), expectedKeyFirstBucket, 1L, nullValue(), null, null); assertBucket(buckets.get(0), expectedKeyFirstBucket, 1L, nullValue(), null, null);
@ -250,7 +250,7 @@ public class DateDerivativeIT extends ESIntegTestCase {
ZoneId timezone = ZoneId.of("CET"); ZoneId timezone = ZoneId.of("CET");
List<IndexRequestBuilder> builders = new ArrayList<>(); List<IndexRequestBuilder> builders = new ArrayList<>();
DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); DateFormatter formatter = DateFormatter.forPattern("uuuu-MM-dd'T'HH:mm:ss").withZone(timezone);
addNTimes(1, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-27T01:00:00")), builders); addNTimes(1, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-27T01:00:00")), builders);
// day with dst shift -1h, 25h long // day with dst shift -1h, 25h long
addNTimes(2, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-28T01:00:00")), builders); addNTimes(2, IDX_DST_END, DateFormatters.from(formatter.parse("2012-10-28T01:00:00")), builders);
@ -274,7 +274,7 @@ public class DateDerivativeIT extends ESIntegTestCase {
List<? extends Bucket> buckets = deriv.getBuckets(); List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(4)); assertThat(buckets.size(), equalTo(4));
DateFormatter dateFormatter = DateFormatter.forPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC); DateFormatter dateFormatter = DateFormatter.forPattern("uuuu-MM-dd").withZone(ZoneOffset.UTC);
ZonedDateTime expectedKeyFirstBucket = ZonedDateTime expectedKeyFirstBucket =
LocalDate.from(dateFormatter.parse("2012-10-27")).atStartOfDay(timezone).withZoneSameInstant(ZoneOffset.UTC); LocalDate.from(dateFormatter.parse("2012-10-27")).atStartOfDay(timezone).withZoneSameInstant(ZoneOffset.UTC);
@ -303,7 +303,7 @@ public class DateDerivativeIT extends ESIntegTestCase {
ZoneId timezone = ZoneId.of("Asia/Kathmandu"); ZoneId timezone = ZoneId.of("Asia/Kathmandu");
List<IndexRequestBuilder> builders = new ArrayList<>(); List<IndexRequestBuilder> builders = new ArrayList<>();
DateFormatter formatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(timezone); DateFormatter formatter = DateFormatter.forPattern("uuuu-MM-dd'T'HH:mm:ss").withZone(timezone);
addNTimes(1, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T22:30:00")), builders); addNTimes(1, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T22:30:00")), builders);
// the shift happens during the next bucket, which includes the 45min that do not start on the full hour // the shift happens during the next bucket, which includes the 45min that do not start on the full hour
addNTimes(2, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T23:30:00")), builders); addNTimes(2, IDX_DST_KATHMANDU, DateFormatters.from(formatter.parse("1985-12-31T23:30:00")), builders);
@ -327,7 +327,7 @@ public class DateDerivativeIT extends ESIntegTestCase {
List<? extends Bucket> buckets = deriv.getBuckets(); List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(4)); assertThat(buckets.size(), equalTo(4));
DateFormatter dateFormatter = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZone(ZoneOffset.UTC); DateFormatter dateFormatter = DateFormatter.forPattern("uuuu-MM-dd'T'HH:mm:ss").withZone(ZoneOffset.UTC);
ZonedDateTime expectedKeyFirstBucket = ZonedDateTime expectedKeyFirstBucket =
LocalDateTime.from(dateFormatter.parse("1985-12-31T22:00:00")).atZone(timezone).withZoneSameInstant(ZoneOffset.UTC); LocalDateTime.from(dateFormatter.parse("1985-12-31T22:00:00")).atZone(timezone).withZoneSameInstant(ZoneOffset.UTC);

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleOriginationDateParser.java
View File

@ -17,7 +17,7 @@ import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_PARSE
public class IndexLifecycleOriginationDateParser { public class IndexLifecycleOriginationDateParser {
private static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("yyyy.MM.dd"); private static final DateFormatter DATE_FORMATTER = DateFormatter.forPattern("uuuu.MM.dd");
private static final String INDEX_NAME_REGEX = "^.*-(\\d{4}.\\d{2}.\\d{2})(-[\\d]+)?$"; private static final String INDEX_NAME_REGEX = "^.*-(\\d{4}.\\d{2}.\\d{2})(-[\\d]+)?$";
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile(INDEX_NAME_REGEX); private static final Pattern INDEX_NAME_PATTERN = Pattern.compile(INDEX_NAME_REGEX);

2
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleOriginationDateParserTests.java
View File

@ -17,7 +17,7 @@ import static org.hamcrest.Matchers.is;
public class IndexLifecycleOriginationDateParserTests extends ESTestCase { public class IndexLifecycleOriginationDateParserTests extends ESTestCase {
private static final DateFormatter dateFormatter = DateFormatter.forPattern("yyyy.MM.dd"); private static final DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
public void testShouldParseIndexNameReturnsFalseWhenOriginationDateIsSet() { public void testShouldParseIndexNameReturnsFalseWhenOriginationDateIsSet() {
Settings settings = Settings.builder() Settings settings = Settings.builder()