honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Core: Rework epoch time parsing for java time (#36914) 

This commit converts the epoch time parsing implementation which uses
the java time api to create DateTimeFormatters instead of DateFormatter
implementations. This will allow multi formats for java time to be
implemented in a single DateTimeFormatter in a future change.
This commit is contained in:
Ryan Ernst 2019-01-07 16:15:30 -08:00 committed by GitHub
parent 56e472bfbc
commit 55d3ca3aa8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 239 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/main/java/org/elasticsearch/common/time/DateFormatters.java
View File

@ -1366,9 +1366,9 @@ public class DateFormatters {
} else if ("yearMonthDay".equals(input) || "year_month_day".equals(input)) {
return YEAR_MONTH_DAY;
} else if ("epoch_second".equals(input)) {
return EpochSecondsDateFormatter.INSTANCE;
return EpochTime.SECONDS_FORMATTER;
} else if ("epoch_millis".equals(input)) {
return EpochMillisDateFormatter.INSTANCE;
return EpochTime.MILLIS_FORMATTER;
// strict date formats here, must be at least 4 digits for year and two for months and two for day
} else if ("strictBasicWeekDate".equals(input) || "strict_basic_week_date".equals(input)) {
return STRICT_BASIC_WEEK_DATE;

113
server/src/main/java/org/elasticsearch/common/time/EpochSecondsDateFormatter.java
View File

@ -1,113 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.time;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.Locale;
import java.util.regex.Pattern;
public class EpochSecondsDateFormatter implements DateFormatter {
public static DateFormatter INSTANCE = new EpochSecondsDateFormatter();
static final DateMathParser DATE_MATH_INSTANCE = new JavaDateMathParser(INSTANCE, INSTANCE);
private static final Pattern SPLIT_BY_DOT_PATTERN = Pattern.compile("\\.");
private EpochSecondsDateFormatter() {}
@Override
public TemporalAccessor parse(String input) {
try {
if (input.contains(".")) {
String[] inputs = SPLIT_BY_DOT_PATTERN.split(input, 2);
Long seconds = Long.valueOf(inputs[0]);
if (inputs[1].length() == 0) {
// this is BWC compatible to joda time, nothing after the dot is allowed
return Instant.ofEpochSecond(seconds, 0).atZone(ZoneOffset.UTC);
}
// scientific notation it is!
if (inputs[1].contains("e")) {
return Instant.ofEpochSecond(Double.valueOf(input).longValue()).atZone(ZoneOffset.UTC);
}
if (inputs[1].length() > 9) {
throw new DateTimeParseException("too much granularity after dot [" + input + "]", input, 0);
}
Long nanos = new BigDecimal(inputs[1]).movePointRight(9 - inputs[1].length()).longValueExact();
if (seconds < 0) {
nanos = nanos * -1;
}
return Instant.ofEpochSecond(seconds, nanos).atZone(ZoneOffset.UTC);
} else {
return Instant.ofEpochSecond(Long.valueOf(input)).atZone(ZoneOffset.UTC);
}
} catch (NumberFormatException e) {
throw new DateTimeParseException("invalid number [" + input + "]", input, 0, e);
}
}
@Override
public String format(TemporalAccessor accessor) {
Instant instant = Instant.from(accessor);
if (instant.getNano() != 0) {
return String.valueOf(instant.getEpochSecond()) + "." + String.valueOf(instant.getNano()).replaceAll("0*$", "");
}
return String.valueOf(instant.getEpochSecond());
}
@Override
public String pattern() {
return "epoch_second";
}
@Override
public Locale locale() {
return Locale.ROOT;
}
@Override
public ZoneId zone() {
return ZoneOffset.UTC;
}
@Override
public DateMathParser toDateMathParser() {
return DATE_MATH_INSTANCE;
}
@Override
public DateFormatter withZone(ZoneId zoneId) {
if (zoneId.equals(ZoneOffset.UTC) == false) {
throw new IllegalArgumentException(pattern() + " date formatter can only be in zone offset UTC");
}
return this;
}
@Override
public DateFormatter withLocale(Locale locale) {
if (Locale.ROOT.equals(locale) == false) {
throw new IllegalArgumentException(pattern() + " date formatter can only be in locale ROOT");
}
return this;
}
}

219
server/src/main/java/org/elasticsearch/common/time/EpochTime.java Normal file
View File

@ -0,0 +1,219 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.time;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.ResolverStyle;
import java.time.format.SignStyle;
import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit;
import java.time.temporal.Temporal;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalField;
import java.time.temporal.TemporalUnit;
import java.time.temporal.ValueRange;
import java.util.Locale;
import java.util.Map;
/**
* This class provides {@link DateTimeFormatter}s capable of parsing epoch seconds and milliseconds.
* <p>
* The seconds formatter is provided by {@link #SECONDS_FORMATTER}.
* The milliseconds formatter is provided by {@link #MILLIS_FORMATTER}.
* <p>
* Both formatters support fractional time, up to nanosecond precision. Values must be positive numbers.
*/
class EpochTime {
private static final ValueRange LONG_POSITIVE_RANGE = ValueRange.of(0, Long.MAX_VALUE);
private static final EpochField SECONDS = new EpochField(ChronoUnit.SECONDS, ChronoUnit.FOREVER, LONG_POSITIVE_RANGE) {
@Override
public boolean isSupportedBy(TemporalAccessor temporal) {
return temporal.isSupported(ChronoField.INSTANT_SECONDS);
}
@Override
public long getFrom(TemporalAccessor temporal) {
return temporal.getLong(ChronoField.INSTANT_SECONDS);
}
@Override
public TemporalAccessor resolve(Map<TemporalField,Long> fieldValues,
TemporalAccessor partialTemporal, ResolverStyle resolverStyle) {
long seconds = fieldValues.remove(this);
fieldValues.put(ChronoField.INSTANT_SECONDS, seconds);
Long nanos = fieldValues.remove(NANOS_OF_SECOND);
if (nanos != null) {
fieldValues.put(ChronoField.NANO_OF_SECOND, nanos);
}
return null;
}
};
private static final EpochField NANOS_OF_SECOND = new EpochField(ChronoUnit.NANOS, ChronoUnit.SECONDS, ValueRange.of(0, 999_999_999)) {
@Override
public boolean isSupportedBy(TemporalAccessor temporal) {
return temporal.isSupported(ChronoField.NANO_OF_SECOND) && temporal.getLong(ChronoField.NANO_OF_SECOND) != 0;
}
@Override
public long getFrom(TemporalAccessor temporal) {
return temporal.getLong(ChronoField.NANO_OF_SECOND);
}
};
private static final EpochField MILLIS = new EpochField(ChronoUnit.MILLIS, ChronoUnit.FOREVER, LONG_POSITIVE_RANGE) {
@Override
public boolean isSupportedBy(TemporalAccessor temporal) {
return temporal.isSupported(ChronoField.INSTANT_SECONDS) && temporal.isSupported(ChronoField.MILLI_OF_SECOND);
}
@Override
public long getFrom(TemporalAccessor temporal) {
return temporal.getLong(ChronoField.INSTANT_SECONDS) * 1_000 + temporal.getLong(ChronoField.MILLI_OF_SECOND);
}
@Override
public TemporalAccessor resolve(Map<TemporalField,Long> fieldValues,
TemporalAccessor partialTemporal, ResolverStyle resolverStyle) {
long secondsAndMillis = fieldValues.remove(this);
long seconds = secondsAndMillis / 1_000;
long nanos = secondsAndMillis % 1000 * 1_000_000;
Long nanosOfMilli = fieldValues.remove(NANOS_OF_MILLI);
if (nanosOfMilli != null) {
nanos += nanosOfMilli;
}
fieldValues.put(ChronoField.INSTANT_SECONDS, seconds);
fieldValues.put(ChronoField.NANO_OF_SECOND, nanos);
return null;
}
};
private static final EpochField NANOS_OF_MILLI = new EpochField(ChronoUnit.NANOS, ChronoUnit.MILLIS, ValueRange.of(0, 999_999)) {
@Override
public boolean isSupportedBy(TemporalAccessor temporal) {
return temporal.isSupported(ChronoField.NANO_OF_SECOND) && temporal.getLong(ChronoField.NANO_OF_SECOND) % 1_000_000 != 0;
}
@Override
public long getFrom(TemporalAccessor temporal) {
return temporal.getLong(ChronoField.NANO_OF_SECOND);
}
};
// this supports seconds without any fraction
private static final DateTimeFormatter SECONDS_FORMATTER1 = new DateTimeFormatterBuilder()
.appendValue(SECONDS, 1, 19, SignStyle.NORMAL)
.toFormatter(Locale.ROOT);
// this supports seconds ending in dot
private static final DateTimeFormatter SECONDS_FORMATTER2 = new DateTimeFormatterBuilder()
.append(SECONDS_FORMATTER1)
.appendLiteral('.')
.toFormatter(Locale.ROOT);
// this supports seconds with a fraction and is also used for printing
private static final DateTimeFormatter SECONDS_FORMATTER3 = new DateTimeFormatterBuilder()
.append(SECONDS_FORMATTER1)
.optionalStart() // optional is used so isSupported will be called when printing
.appendFraction(NANOS_OF_SECOND, 1, 9, true)
.optionalEnd()
.toFormatter(Locale.ROOT);
// this supports milliseconds without any fraction
private static final DateTimeFormatter MILLISECONDS_FORMATTER1 = new DateTimeFormatterBuilder()
.appendValue(MILLIS, 1, 19, SignStyle.NORMAL)
.toFormatter(Locale.ROOT);
// this supports milliseconds ending in dot
private static final DateTimeFormatter MILLISECONDS_FORMATTER2 = new DateTimeFormatterBuilder()
.append(MILLISECONDS_FORMATTER1)
.appendLiteral('.')
.toFormatter(Locale.ROOT);
// this supports milliseconds with a fraction and is also used for printing
private static final DateTimeFormatter MILLISECONDS_FORMATTER3 = new DateTimeFormatterBuilder()
.append(MILLISECONDS_FORMATTER1)
.optionalStart() // optional is used so isSupported will be called when printing
.appendFraction(NANOS_OF_MILLI, 1, 6, true)
.optionalEnd()
.toFormatter(Locale.ROOT);
static final DateFormatter SECONDS_FORMATTER = new JavaDateFormatter("epoch_second", SECONDS_FORMATTER3,
SECONDS_FORMATTER1, SECONDS_FORMATTER2, SECONDS_FORMATTER3);
static final DateFormatter MILLIS_FORMATTER = new JavaDateFormatter("epoch_millis", MILLISECONDS_FORMATTER3,
MILLISECONDS_FORMATTER1, MILLISECONDS_FORMATTER2, MILLISECONDS_FORMATTER3);
private abstract static class EpochField implements TemporalField {
private final TemporalUnit baseUnit;
private final TemporalUnit rangeUnit;
private final ValueRange range;
private EpochField(TemporalUnit baseUnit, TemporalUnit rangeUnit, ValueRange range) {
this.baseUnit = baseUnit;
this.rangeUnit = rangeUnit;
this.range = range;
}
@Override
public String getDisplayName(Locale locale) {
return toString();
}
@Override
public String toString() {
return "Epoch" + baseUnit.toString() + (rangeUnit != ChronoUnit.FOREVER ? "Of" + rangeUnit.toString() : "");
}
@Override
public TemporalUnit getBaseUnit() {
return baseUnit;
}
@Override
public TemporalUnit getRangeUnit() {
return rangeUnit;
}
@Override
public ValueRange range() {
return range;
}
@Override
public boolean isDateBased() {
return false;
}
@Override
public boolean isTimeBased() {
return true;
}
@Override
public ValueRange rangeRefinedBy(TemporalAccessor temporal) {
return range();
}
@SuppressWarnings("unchecked")
@Override
public <R extends Temporal> R adjustInto(R temporal, long newValue) {
return (R) temporal.with(this, newValue);
}
}
}

11
server/src/test/java/org/elasticsearch/common/joda/JavaJodaTimeDuellingTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common.joda;
import org.elasticsearch.bootstrap.JavaVersion;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.test.ESTestCase;
@ -384,6 +385,7 @@ public class JavaJodaTimeDuellingTests extends ESTestCase {
ZonedDateTime javaDate = ZonedDateTime.of(year, month, day, hour, minute, second, 0, ZoneOffset.UTC);
DateTime jodaDate = new DateTime(year, month, day, hour, minute, second, DateTimeZone.UTC);
assertSamePrinterOutput("epoch_second", javaDate, jodaDate);
assertSamePrinterOutput("basicDate", javaDate, jodaDate);
assertSamePrinterOutput("basicDateTime", javaDate, jodaDate);
@ -428,7 +430,7 @@ public class JavaJodaTimeDuellingTests extends ESTestCase {
assertSamePrinterOutput("year", javaDate, jodaDate);
assertSamePrinterOutput("yearMonth", javaDate, jodaDate);
assertSamePrinterOutput("yearMonthDay", javaDate, jodaDate);
assertSamePrinterOutput("epoch_second", javaDate, jodaDate);
assertSamePrinterOutput("epoch_millis", javaDate, jodaDate);
assertSamePrinterOutput("strictBasicWeekDate", javaDate, jodaDate);
assertSamePrinterOutput("strictBasicWeekDateTime", javaDate, jodaDate);
@ -476,6 +478,12 @@ public class JavaJodaTimeDuellingTests extends ESTestCase {
assertThat(jodaDate.getMillis(), is(javaDate.toInstant().toEpochMilli()));
String javaTimeOut = DateFormatters.forPattern(format).format(javaDate);
String jodaTimeOut = DateFormatter.forPattern(format).formatJoda(jodaDate);
if (JavaVersion.current().getVersion().get(0) == 8 && javaTimeOut.endsWith(".0")
&& (format.equals("epoch_second") || format.equals("epoch_millis"))) {
// java 8 has a bug in DateTimeFormatter usage when printing dates that rely on isSupportedBy for fields, which is
// what we use for epoch time. This change accounts for that bug. It should be removed when java 8 support is removed
jodaTimeOut += ".0";
}
String message = String.format(Locale.ROOT, "expected string representation to be equal for format [%s]: joda [%s], java [%s]",
format, jodaTimeOut, javaTimeOut);
assertThat(message, javaTimeOut, is(jodaTimeOut));
@ -484,7 +492,6 @@ public class JavaJodaTimeDuellingTests extends ESTestCase {
private void assertSameDate(String input, String format) {
DateFormatter jodaFormatter = Joda.forPattern(format);
DateFormatter javaFormatter = DateFormatters.forPattern(format);
assertSameDate(input, format, jodaFormatter, javaFormatter);
}

1
server/src/test/java/org/elasticsearch/common/joda/JodaTests.java
View File

@ -29,7 +29,6 @@ import java.time.ZoneOffset;
public class JodaTests extends ESTestCase {
public void testBasicTTimePattern() {
DateFormatter formatter1 = DateFormatter.forPattern("basic_t_time");
assertEquals(formatter1.pattern(), "basic_t_time");

55
server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java
View File

@ -23,7 +23,6 @@ import org.elasticsearch.test.ESTestCase;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.Locale;
@ -58,21 +57,6 @@ public class DateFormattersTests extends ESTestCase {
assertThat(instant.getEpochSecond(), is(12L));
assertThat(instant.getNano(), is(345_000_000));
}
{
Instant instant = Instant.from(formatter.parse("-12345.6789"));
assertThat(instant.getEpochSecond(), is(-13L));
assertThat(instant.getNano(), is(1_000_000_000 - 345_678_900));
}
{
Instant instant = Instant.from(formatter.parse("-436134.241272"));
assertThat(instant.getEpochSecond(), is(-437L));
assertThat(instant.getNano(), is(1_000_000_000 - 134_241_272));
}
{
Instant instant = Instant.from(formatter.parse("-12345"));
assertThat(instant.getEpochSecond(), is(-13L));
assertThat(instant.getNano(), is(1_000_000_000 - 345_000_000));
}
{
Instant instant = Instant.from(formatter.parse("0"));
assertThat(instant.getEpochSecond(), is(0L));
@ -83,10 +67,10 @@ public class DateFormattersTests extends ESTestCase {
public void testEpochMilliParser() {
DateFormatter formatter = DateFormatters.forPattern("epoch_millis");
DateTimeParseException e = expectThrows(DateTimeParseException.class, () -> formatter.parse("invalid"));
assertThat(e.getMessage(), containsString("invalid number"));
assertThat(e.getMessage(), containsString("could not be parsed"));
e = expectThrows(DateTimeParseException.class, () -> formatter.parse("123.1234567"));
assertThat(e.getMessage(), containsString("too much granularity after dot [123.1234567]"));
assertThat(e.getMessage(), containsString("unparsed text found at index 3"));
}
// this is not in the duelling tests, because the epoch second parser in joda time drops the milliseconds after the comma
@ -108,17 +92,14 @@ public class DateFormattersTests extends ESTestCase {
assertThat(Instant.from(formatter.parse("1234.12345678")).getNano(), is(123_456_780));
assertThat(Instant.from(formatter.parse("1234.123456789")).getNano(), is(123_456_789));
assertThat(Instant.from(formatter.parse("-1234.567")).toEpochMilli(), is(-1234567L));
assertThat(Instant.from(formatter.parse("-1234")).getNano(), is(0));
DateTimeParseException e = expectThrows(DateTimeParseException.class, () -> formatter.parse("1234.1234567890"));
assertThat(e.getMessage(), is("too much granularity after dot [1234.1234567890]"));
assertThat(e.getMessage(), is("Text '1234.1234567890' could not be parsed, unparsed text found at index 4"));
e = expectThrows(DateTimeParseException.class, () -> formatter.parse("1234.123456789013221"));
assertThat(e.getMessage(), is("too much granularity after dot [1234.123456789013221]"));
assertThat(e.getMessage(), is("Text '1234.123456789013221' could not be parsed, unparsed text found at index 4"));
e = expectThrows(DateTimeParseException.class, () -> formatter.parse("abc"));
assertThat(e.getMessage(), is("invalid number [abc]"));
assertThat(e.getMessage(), is("Text 'abc' could not be parsed at index 0"));
e = expectThrows(DateTimeParseException.class, () -> formatter.parse("1234.abc"));
assertThat(e.getMessage(), is("invalid number [1234.abc]"));
assertThat(e.getMessage(), is("Text '1234.abc' could not be parsed, unparsed text found at index 4"));
}
public void testEpochMilliParsersWithDifferentFormatters() {
@ -132,18 +113,6 @@ public class DateFormattersTests extends ESTestCase {
assertThat(DateFormatters.forPattern("strict_date_optional_time").locale(), is(Locale.ROOT));
Locale locale = randomLocale(random());
assertThat(DateFormatters.forPattern("strict_date_optional_time").withLocale(locale).locale(), is(locale));
if (locale.equals(Locale.ROOT)) {
DateFormatter millisFormatter = DateFormatters.forPattern("epoch_millis");
assertThat(millisFormatter.withLocale(locale), is(millisFormatter));
DateFormatter secondFormatter = DateFormatters.forPattern("epoch_second");
assertThat(secondFormatter.withLocale(locale), is(secondFormatter));
} else {
IllegalArgumentException e =
expectThrows(IllegalArgumentException.class, () -> DateFormatters.forPattern("epoch_millis").withLocale(locale));
assertThat(e.getMessage(), is("epoch_millis date formatter can only be in locale ROOT"));
e = expectThrows(IllegalArgumentException.class, () -> DateFormatters.forPattern("epoch_second").withLocale(locale));
assertThat(e.getMessage(), is("epoch_second date formatter can only be in locale ROOT"));
}
}
public void testTimeZones() {
@ -151,18 +120,6 @@ public class DateFormattersTests extends ESTestCase {
assertThat(DateFormatters.forPattern("strict_date_optional_time").zone(), is(nullValue()));
ZoneId zoneId = randomZone();
assertThat(DateFormatters.forPattern("strict_date_optional_time").withZone(zoneId).zone(), is(zoneId));
if (zoneId.equals(ZoneOffset.UTC)) {
DateFormatter millisFormatter = DateFormatters.forPattern("epoch_millis");
assertThat(millisFormatter.withZone(zoneId), is(millisFormatter));
DateFormatter secondFormatter = DateFormatters.forPattern("epoch_second");
assertThat(secondFormatter.withZone(zoneId), is(secondFormatter));
} else {
IllegalArgumentException e =
expectThrows(IllegalArgumentException.class, () -> DateFormatters.forPattern("epoch_millis").withZone(zoneId));
assertThat(e.getMessage(), is("epoch_millis date formatter can only be in zone offset UTC"));
e = expectThrows(IllegalArgumentException.class, () -> DateFormatters.forPattern("epoch_second").withZone(zoneId));
assertThat(e.getMessage(), is("epoch_second date formatter can only be in zone offset UTC"));
}
}
public void testEqualsAndHashcode() {

4
server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -897,8 +898,9 @@ public class SearchFieldsIT extends ESIntegTestCase {
assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo("4.0"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo("5.0"));
assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo("6.0"));
// TODO: switch to java date formatter, but will require special casing java 8 as there is a bug with epoch formatting there
assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(),
equalTo(DateFormatters.forPattern("epoch_millis").format(date)));
equalTo(Joda.forPattern("epoch_millis").format(date)));
}
public void testScriptFields() throws Exception {