From 8a4c3cab84253308c2f7fa4eda4a4221c73545e2 Mon Sep 17 00:00:00 2001 From: Arek Burdach Date: Wed, 20 Jan 2021 15:28:29 +0100 Subject: [PATCH] NIFI-8161 Migrated EL from SimpleDateFormat to DateTimeFormatter This closes #7169 Signed-off-by: David Handermann --- .../language/antlr/AttributeExpressionLexer.g | 2 +- .../evaluation/cast/DateCastEvaluator.java | 28 +++---- .../evaluation/functions/FormatEvaluator.java | 55 ++++++++++--- .../functions/StringToDateEvaluator.java | 57 ++++++++++--- .../expression/language/TestQuery.java | 53 +++++++++--- .../org/apache/nifi/util/FormatUtils.java | 47 +++++++++++ .../nifi/processor/TestFormatUtils.java | 81 +++++++++++++++++++ .../asciidoc/expression-language-guide.adoc | 10 +-- 8 files changed, 281 insertions(+), 52 deletions(-) diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g index 02901c4b4e..3edcf381b3 100644 --- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g +++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g @@ -171,7 +171,7 @@ GREATER_THAN : 'gt'; LESS_THAN : 'lt'; GREATER_THAN_OR_EQUAL : 'ge'; LESS_THAN_OR_EQUAL : 'le'; -FORMAT : 'format'; // takes string date format; uses SimpleDateFormat +FORMAT : 'format'; // takes string date format; uses DateTimeFormatter FORMAT_INSTANT : 'formatInstant'; TO_DATE : 'toDate'; // takes string date format; converts the subject to a Long based on the date format TO_INSTANT : 'toInstant'; diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java index b7c4428e6d..cf729f3f5d 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DateCastEvaluator.java @@ -26,21 +26,25 @@ import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResul import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException; import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException; import org.apache.nifi.expression.AttributeExpression.ResultType; +import org.apache.nifi.util.FormatUtils; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.Date; -import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; public class DateCastEvaluator extends DateEvaluator { public static final String DATE_TO_STRING_FORMAT = "EEE MMM dd HH:mm:ss zzz yyyy"; + public static final DateTimeFormatter DATE_TO_STRING_FORMATTER = FormatUtils.prepareLenientCaseInsensitiveDateTimeFormatter(DATE_TO_STRING_FORMAT); public static final Pattern DATE_TO_STRING_PATTERN = Pattern.compile("(?:[a-zA-Z]{3} ){2}\\d{2} \\d{2}\\:\\d{2}\\:\\d{2} (?:.*?) \\d{4}"); public static final String ALTERNATE_FORMAT_WITHOUT_MILLIS = "yyyy/MM/dd HH:mm:ss"; public static final String ALTERNATE_FORMAT_WITH_MILLIS = "yyyy/MM/dd HH:mm:ss.SSS"; + public static final DateTimeFormatter ALTERNATE_FORMATTER_WITHOUT_MILLIS = FormatUtils.prepareLenientCaseInsensitiveDateTimeFormatter(ALTERNATE_FORMAT_WITHOUT_MILLIS); + public static final DateTimeFormatter ALTERNATE_FORMATTER_WITH_MILLIS = FormatUtils.prepareLenientCaseInsensitiveDateTimeFormatter(ALTERNATE_FORMAT_WITH_MILLIS); public static final Pattern ALTERNATE_PATTERN = Pattern.compile("\\d{4}/\\d{2}/\\d{2} \\d{2}\\:\\d{2}\\:\\d{2}(\\.\\d{3})?"); public static final Pattern NUMBER_PATTERN = Pattern.compile("\\d+"); @@ -68,12 +72,10 @@ public class DateCastEvaluator extends DateEvaluator { case STRING: final String value = ((StringQueryResult) result).getValue().trim(); if (DATE_TO_STRING_PATTERN.matcher(value).matches()) { - final SimpleDateFormat sdf = new SimpleDateFormat(DATE_TO_STRING_FORMAT, Locale.US); - try { - final Date date = sdf.parse(value); + final Date date = Date.from(DATE_TO_STRING_FORMATTER.parse(value, Instant::from)); return new DateQueryResult(date); - } catch (final ParseException pe) { + } catch (final DateTimeParseException pe) { final String details = "Format: '" + DATE_TO_STRING_FORMAT + "' Value: '" + value + "'"; throw new AttributeExpressionLanguageException("Could not parse date using " + details, pe); } @@ -84,19 +86,17 @@ public class DateCastEvaluator extends DateEvaluator { if (altMatcher.matches()) { final String millisValue = altMatcher.group(1); - final String format; + final DateTimeFormatter formatter; if (millisValue == null) { - format = ALTERNATE_FORMAT_WITHOUT_MILLIS; + formatter = ALTERNATE_FORMATTER_WITHOUT_MILLIS; } else { - format = ALTERNATE_FORMAT_WITH_MILLIS; + formatter = ALTERNATE_FORMATTER_WITH_MILLIS; } - final SimpleDateFormat sdf = new SimpleDateFormat(format, Locale.US); - try { - final Date date = sdf.parse(value); + final Date date = Date.from(FormatUtils.parseToInstant(formatter, value)); return new DateQueryResult(date); - } catch (final ParseException pe) { + } catch (final DateTimeParseException pe) { throw new AttributeExpressionLanguageException("Could not parse input as date", pe); } } else { diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java index 1daa427d37..435944043d 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java @@ -17,16 +17,20 @@ package org.apache.nifi.attribute.expression.language.evaluation.functions; import org.apache.nifi.attribute.expression.language.EvaluationContext; +import org.apache.nifi.attribute.expression.language.StandardEvaluationContext; import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.literals.StringLiteralEvaluator; -import java.text.SimpleDateFormat; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; import java.util.Date; import java.util.Locale; -import java.util.TimeZone; public class FormatEvaluator extends StringEvaluator { @@ -34,9 +38,31 @@ public class FormatEvaluator extends StringEvaluator { private final Evaluator format; private final Evaluator timeZone; + private final DateTimeFormatter preparedFormatter; + private final boolean preparedFormatterHasRequestedTimeZone; + public FormatEvaluator(final DateEvaluator subject, final Evaluator format, final Evaluator timeZone) { this.subject = subject; this.format = format; + // if the search string is a literal, we don't need to prepare formatter each time; we can just + // prepare it once. Otherwise, it must be prepared for each time. + if (format instanceof StringLiteralEvaluator) { + String formatPattern = format.evaluate(new StandardEvaluationContext(Collections.emptyMap())).getValue(); + DateTimeFormatter dtf = DateTimeFormatter.ofPattern(formatPattern, Locale.US); + if (timeZone == null) { + preparedFormatter = dtf; + preparedFormatterHasRequestedTimeZone = true; + } else if (timeZone instanceof StringLiteralEvaluator) { + preparedFormatter = dtf.withZone(ZoneId.of(timeZone.evaluate(new StandardEvaluationContext(Collections.emptyMap())).getValue())); + preparedFormatterHasRequestedTimeZone = true; + } else { + preparedFormatter = dtf; + preparedFormatterHasRequestedTimeZone = false; + } + } else { + preparedFormatter = null; + preparedFormatterHasRequestedTimeZone = false; + } this.timeZone = timeZone; } @@ -47,23 +73,28 @@ public class FormatEvaluator extends StringEvaluator { return new StringQueryResult(null); } - final QueryResult formatResult = format.evaluate(evaluationContext); - final String format = formatResult.getValue(); - if (format == null) { - return null; + DateTimeFormatter dtf; + if (preparedFormatter == null) { + final QueryResult formatResult = format.evaluate(evaluationContext); + final String format = formatResult.getValue(); + if (format == null) { + return null; + } + dtf = DateTimeFormatter.ofPattern(format, Locale.US); + } else { + dtf = preparedFormatter; } - final SimpleDateFormat sdf = new SimpleDateFormat(format, Locale.US); - - if(timeZone != null) { + if ((preparedFormatter == null || !preparedFormatterHasRequestedTimeZone) && timeZone != null) { final QueryResult tzResult = timeZone.evaluate(evaluationContext); final String tz = tzResult.getValue(); - if(tz != null && TimeZone.getTimeZone(tz) != null) { - sdf.setTimeZone(TimeZone.getTimeZone(tz)); + if (tz != null) { + dtf = dtf.withZone(ZoneId.of(tz)); } } - return new StringQueryResult(sdf.format(subjectValue)); + ZonedDateTime subjectDateTime = subjectValue.toInstant().atZone(ZoneId.systemDefault()); + return new StringQueryResult(dtf.format(subjectDateTime)); } @Override diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java index f20c0840e4..9cbb939341 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java @@ -17,17 +17,20 @@ package org.apache.nifi.attribute.expression.language.evaluation.functions; import org.apache.nifi.attribute.expression.language.EvaluationContext; +import org.apache.nifi.attribute.expression.language.StandardEvaluationContext; import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult; import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.literals.StringLiteralEvaluator; import org.apache.nifi.attribute.expression.language.exception.IllegalAttributeException; +import org.apache.nifi.util.FormatUtils; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Collections; import java.util.Date; -import java.util.Locale; -import java.util.TimeZone; public class StringToDateEvaluator extends DateEvaluator { @@ -35,9 +38,31 @@ public class StringToDateEvaluator extends DateEvaluator { private final Evaluator format; private final Evaluator timeZone; + private final DateTimeFormatter preparedFormatter; + private final boolean preparedFormatterHasRequestedTimeZone; + public StringToDateEvaluator(final Evaluator subject, final Evaluator format, final Evaluator timeZone) { this.subject = subject; this.format = format; + // if the search string is a literal, we don't need to prepare formatter each time; we can just + // prepare it once. Otherwise, it must be prepared for each time. + if (format instanceof StringLiteralEvaluator) { + String evaluatedFormat = format.evaluate(new StandardEvaluationContext(Collections.emptyMap())).getValue(); + DateTimeFormatter dtf = FormatUtils.prepareLenientCaseInsensitiveDateTimeFormatter(evaluatedFormat); + if (timeZone == null) { + preparedFormatter = dtf; + preparedFormatterHasRequestedTimeZone = true; + } else if (timeZone instanceof StringLiteralEvaluator) { + preparedFormatter = dtf.withZone(ZoneId.of(timeZone.evaluate(new StandardEvaluationContext(Collections.emptyMap())).getValue())); + preparedFormatterHasRequestedTimeZone = true; + } else { + preparedFormatter = dtf; + preparedFormatterHasRequestedTimeZone = false; + } + } else { + preparedFormatter = null; + preparedFormatterHasRequestedTimeZone = false; + } this.timeZone = timeZone; } @@ -49,21 +74,31 @@ public class StringToDateEvaluator extends DateEvaluator { return new DateQueryResult(null); } - final SimpleDateFormat sdf = new SimpleDateFormat(formatValue, Locale.US); + DateTimeFormatter dtf; + if (preparedFormatter != null) { + dtf = preparedFormatter; + } else { + final QueryResult formatResult = format.evaluate(evaluationContext); + final String format = formatResult.getValue(); + if (format == null) { + return null; + } + dtf = FormatUtils.prepareLenientCaseInsensitiveDateTimeFormatter(format); + } - if(timeZone != null) { + if ((preparedFormatter == null || !preparedFormatterHasRequestedTimeZone) && timeZone != null) { final QueryResult tzResult = timeZone.evaluate(evaluationContext); final String tz = tzResult.getValue(); - if(tz != null && TimeZone.getTimeZone(tz) != null) { - sdf.setTimeZone(TimeZone.getTimeZone(tz)); + if(tz != null) { + dtf = dtf.withZone(ZoneId.of(tz)); } } try { - return new DateQueryResult(sdf.parse(subjectValue)); - } catch (final ParseException e) { + return new DateQueryResult(Date.from(FormatUtils.parseToInstant(dtf, subjectValue))); + } catch (final DateTimeParseException e) { throw new IllegalAttributeException("Cannot parse attribute value as a date; date format: " - + formatValue + "; attribute value: " + subjectValue); + + formatValue + "; attribute value: " + subjectValue + ". Error: " + e.getMessage()); } catch (final IllegalArgumentException e) { throw new IllegalAttributeException("Invalid date format: " + formatValue); } diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java index 23ea9fb359..43ad4c3e08 100644 --- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java +++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java @@ -37,7 +37,6 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; -import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -52,6 +51,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; import static java.lang.Double.NEGATIVE_INFINITY; @@ -387,10 +387,9 @@ public class TestQuery { // the date.toString() above will end up truncating the milliseconds. So remove millis from the Date before // formatting it - final SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS", Locale.US); - final long millis = date.getTime() % 1000L; - final Date roundedToNearestSecond = new Date(date.getTime() - millis); - final String formatted = sdf.format(roundedToNearestSecond); + final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS", Locale.US); + final Instant truncatedSecond = date.toInstant().truncatedTo(ChronoUnit.SECONDS); + final String formatted = dtf.format(truncatedSecond); final QueryResult result = query.evaluate(new StandardEvaluationContext(attributes)); assertEquals(ResultType.STRING, result.getResultType()); @@ -777,7 +776,7 @@ public class TestQuery { @Test public void testProblematic1() { // There was a bug that prevented this expression from compiling. This test just verifies that it now compiles. - final String queryString = "${xx:append( \"120101\" ):toDate( 'yyMMddHHmmss' ):format( \"yy-MM-dd’T’HH:mm:ss\") }"; + final String queryString = "${xx:append( \"120101\" ):toDate( 'yyMMddHHmmss' ):format( \"yy-MM-dd'T'HH:mm:ss\") }"; Query.compile(queryString); } @@ -944,11 +943,18 @@ public class TestQuery { final String format = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; final String query = "startDateTime=\"${date:toNumber():toDate():format(\"" + format + "\")}\""; - final String result = Query.evaluateExpressions(query, attributes, null); - final String expectedTime = new SimpleDateFormat(format, Locale.US).format(timestamp); - assertEquals("startDateTime=\"" + expectedTime + "\"", result); + TimeZone current = TimeZone.getDefault(); + TimeZone defaultTimeZone = TimeZone.getTimeZone("Europe/Kiev"); + TimeZone.setDefault(defaultTimeZone); + try { + final String result = Query.evaluateExpressions(query, attributes, null); + final String expectedTime = DateTimeFormatter.ofPattern(format, Locale.US).format(Instant.ofEpochMilli(timestamp).atZone(defaultTimeZone.toZoneId())); + assertEquals("startDateTime=\"" + expectedTime + "\"", result); + } finally { + TimeZone.setDefault(current); + } final List ranges = Query.extractExpressionRanges(query); assertEquals(1, ranges.size()); } @@ -973,6 +979,28 @@ public class TestQuery { assertEquals(1, ranges.size()); } + @Test + public void testFormatUsesLocalTimeZoneUnlessIsSpecified() { + TimeZone current = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Kiev")); + try { + final String formatWithZoneInvocation = "format(\"yyyy-MM-dd HH:mm:ss\", \"GMT\")"; + assertEquals("2020-01-01 00:00:00", evaluateFormatDate("2020-01-01 00:00:00", formatWithZoneInvocation)); + + final String formatWithoutZoneInvocation = "format(\"yyyy-MM-dd HH:mm:ss\")"; + assertEquals("2020-02-01 02:00:00", evaluateFormatDate("2020-02-01 00:00:00", formatWithoutZoneInvocation)); + } finally { + TimeZone.setDefault(current); + } + } + + private String evaluateFormatDate(String givenDateStringInGMT, String formatInvocation) { + final Map attributes = new HashMap<>(); + attributes.put("date", String.valueOf(givenDateStringInGMT)); + final String query = "${date:toDate(\"yyyy-MM-dd HH:mm:ss\", \"GMT\"):" + formatInvocation + "}"; + return Query.evaluateExpressions(query, attributes, null); + } + @Test public void testDateConversion() { final Map attributes = new HashMap<>(); @@ -1935,6 +1963,13 @@ public class TestQuery { verifyEquals("${blue:toDate('yyyyMMddHHmmss'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\")}", attributes, "2013/09/17 16:26:43.000Z"); } + @Test + public void testDateFormatConversionIsCaseInsensitive() { + final Map attributes = new HashMap<>(); + attributes.put("blue", "10may2004"); + verifyEquals("${blue:toDate('ddMMMyyyy'):format('yyyy/MM/dd')}", attributes, "2004/05/10"); + } + @Test public void testDateFormatConversionWithTimeZone() { final Map attributes = new HashMap<>(); diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java index d7fdf5f10f..33bf24b8cf 100644 --- a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java +++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java @@ -17,8 +17,17 @@ package org.apache.nifi.util; import java.text.NumberFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.TemporalAccessor; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -46,6 +55,8 @@ public class FormatUtils { public static final Pattern TIME_DURATION_PATTERN = Pattern.compile(TIME_DURATION_REGEX); private static final List TIME_UNIT_MULTIPLIERS = Arrays.asList(1000L, 1000L, 1000L, 60L, 60L, 24L); + private static final LocalDate EPOCH_INITIAL_DATE = LocalDate.of(1970, 1, 1); + /** * Formats the specified count by adding commas. * @@ -425,4 +436,40 @@ public class FormatUtils { return sb.toString(); } + + public static DateTimeFormatter prepareLenientCaseInsensitiveDateTimeFormatter(String pattern) { + return new DateTimeFormatterBuilder() + .parseLenient() + .parseCaseInsensitive() + .appendPattern(pattern) + .toFormatter(Locale.US); + } + + /** + * Parse text to Instant - support different formats like: zoned date time, date time, date, time (similar to those supported in SimpleDateFormat) + * @param formatter configured formatter + * @param text text which will be parsed + * @return parsed Instant + */ + public static Instant parseToInstant(DateTimeFormatter formatter, String text) { + if (text == null) { + throw new IllegalArgumentException("Text cannot be null"); + } + + TemporalAccessor parsed = formatter.parseBest(text, Instant::from, LocalDateTime::from, LocalDate::from, LocalTime::from); + if (parsed instanceof Instant) { + return (Instant) parsed; + } else if (parsed instanceof LocalDateTime) { + return toInstantInSystemDefaultTimeZone((LocalDateTime) parsed); + } else if (parsed instanceof LocalDate) { + return toInstantInSystemDefaultTimeZone(((LocalDate) parsed).atTime(0, 0)); + } else { + return toInstantInSystemDefaultTimeZone(((LocalTime) parsed).atDate(EPOCH_INITIAL_DATE)); + } + } + + private static Instant toInstantInSystemDefaultTimeZone(LocalDateTime dateTime) { + return dateTime.atZone(ZoneId.systemDefault()).toInstant(); + } + } diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/processor/TestFormatUtils.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/processor/TestFormatUtils.java index 86599b6c87..f95c0582d5 100644 --- a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/processor/TestFormatUtils.java +++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/processor/TestFormatUtils.java @@ -20,12 +20,23 @@ import org.apache.nifi.util.FormatUtils; import org.junit.jupiter.api.Test; import java.text.DecimalFormatSymbols; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; public class TestFormatUtils { + private static String NEW_YORK_TIME_ZONE_ID = "America/New_York"; + private static String KIEV_TIME_ZONE_ID = "Europe/Kiev"; + @Test public void testParse() { assertEquals(3, FormatUtils.getTimeDuration("3000 ms", TimeUnit.SECONDS)); @@ -103,4 +114,74 @@ public class TestFormatUtils { assertEquals(String.format("181%s9 TB", decimalFormatSymbols.getDecimalSeparator()), FormatUtils.formatDataSize(200_000_000_000_000d)); } + + @Test + public void testParseToInstantUsingFormatterWithoutZones() throws Exception { + // GMT- + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss", "2020-01-01 02:00:00", NEW_YORK_TIME_ZONE_ID, null, "2020-01-01T07:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd", "2020-01-01", NEW_YORK_TIME_ZONE_ID, null, "2020-01-01T05:00:00"); + checkSameResultsWithSimpleDateFormat("HH:mm:ss", "03:00:00", NEW_YORK_TIME_ZONE_ID, null, "1970-01-01T08:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MMM-dd", "2020-may-01", NEW_YORK_TIME_ZONE_ID, null, "2020-05-01T04:00:00"); + + // GMT+ + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss", "2020-01-01 02:00:00", KIEV_TIME_ZONE_ID, null, "2020-01-01T00:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd", "2020-01-01", KIEV_TIME_ZONE_ID, null, "2019-12-31T22:00:00"); + checkSameResultsWithSimpleDateFormat("HH:mm:ss", "03:00:00", KIEV_TIME_ZONE_ID, null, "1970-01-01T00:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MMM-dd", "2020-may-01", KIEV_TIME_ZONE_ID, null, "2020-04-30T21:00:00"); + + // UTC + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss", "2020-01-01 02:00:00", ZoneOffset.UTC.getId(), null, "2020-01-01T02:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd", "2020-01-01", ZoneOffset.UTC.getId(), null, "2020-01-01T00:00:00"); + checkSameResultsWithSimpleDateFormat("HH:mm:ss", "03:00:00", ZoneOffset.UTC.getId(), null, "1970-01-01T03:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MMM-dd", "2020-may-01", ZoneOffset.UTC.getId(), null, "2020-05-01T00:00:00"); + } + + @Test + public void testParseToInstantUsingFormatterWithZone() throws Exception { + for (String systemDefaultZoneId : new String[]{ NEW_YORK_TIME_ZONE_ID, KIEV_TIME_ZONE_ID, ZoneOffset.UTC.getId()}) { + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss", "2020-01-01 02:00:00", systemDefaultZoneId, NEW_YORK_TIME_ZONE_ID, "2020-01-01T07:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss", "2020-01-01 02:00:00", systemDefaultZoneId, KIEV_TIME_ZONE_ID, "2020-01-01T00:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss", "2020-01-01 02:00:00", systemDefaultZoneId, ZoneOffset.UTC.getId(), "2020-01-01T02:00:00"); + } + } + + @Test + public void testParseToInstantWithZonePassedInText() throws Exception { + for (String systemDefaultZoneId : new String[]{ NEW_YORK_TIME_ZONE_ID, KIEV_TIME_ZONE_ID, ZoneOffset.UTC.getId()}) { + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", "2020-01-01 02:00:00 -0100", systemDefaultZoneId, null, "2020-01-01T03:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", "2020-01-01 02:00:00 +0100", systemDefaultZoneId, null, "2020-01-01T01:00:00"); + checkSameResultsWithSimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", "2020-01-01 02:00:00 +0000", systemDefaultZoneId, null, "2020-01-01T02:00:00"); + } + } + + private void checkSameResultsWithSimpleDateFormat(String pattern, String parsedDateTime, String systemDefaultZoneId, String formatZoneId, String expectedUtcDateTime) throws Exception { + TimeZone current = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone(systemDefaultZoneId)); + try { + checkSameResultsWithSimpleDateFormat(pattern, parsedDateTime, formatZoneId, expectedUtcDateTime); + } finally { + TimeZone.setDefault(current); + } + } + + private void checkSameResultsWithSimpleDateFormat(String pattern, String parsedDateTime, String formatterZoneId, String expectedUtcDateTime) throws Exception { + Instant expectedInstant = LocalDateTime.parse(expectedUtcDateTime).atZone(ZoneOffset.UTC).toInstant(); + + // reference implementation + SimpleDateFormat sdf = new SimpleDateFormat(pattern, Locale.US); + if (formatterZoneId != null) { + sdf.setTimeZone(TimeZone.getTimeZone(formatterZoneId)); + } + Instant simpleDateFormatResult = sdf.parse(parsedDateTime).toInstant(); + assertEquals(expectedInstant, simpleDateFormatResult); + + // current implementation + DateTimeFormatter dtf = FormatUtils.prepareLenientCaseInsensitiveDateTimeFormatter(pattern); + if (formatterZoneId != null) { + dtf = dtf.withZone(ZoneId.of(formatterZoneId)); + } + Instant result = FormatUtils.parseToInstant(dtf, parsedDateTime); + assertEquals(expectedInstant, result); + } + } diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc index c592feb743..bde6780f26 100644 --- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc +++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc @@ -261,7 +261,7 @@ Language supports four different data types: - *Date*: A Date is an object that holds a Date and Time. Utilizing the <> and <> functions this data type can be converted to/from Strings and numbers. If the whole Expression Language expression is evaluated to be a date then it will be converted to a String with the format: " ::