From 3bb876eb835499845e66848e815cd166b683232c Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Tue, 3 Jan 2017 13:16:40 +0100 Subject: [PATCH] NIFI-2908 Added TimeZone to toDate() and format() EL This closes #1381. Signed-off-by: Koji Kawamura --- .../antlr/AttributeExpressionParser.g | 8 +++---- .../attribute/expression/language/Query.java | 14 ++++++++++--- .../evaluation/functions/FormatEvaluator.java | 17 +++++++++++++-- .../functions/StringToDateEvaluator.java | 17 +++++++++++++-- .../expression/language/TestQuery.java | 12 +++++++++-- .../asciidoc/expression-language-guide.adoc | 21 ++++++++++++------- 6 files changed, 68 insertions(+), 21 deletions(-) diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g index 576f0118cb..4d48ee093f 100644 --- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g +++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g @@ -75,10 +75,10 @@ tokens { // functions that return Strings zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!; oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY | - PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) | + PREPEND | APPEND | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) | (TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!); twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL | IF_ELSE) LPAREN! anyArg COMMA! anyArg RPAREN!) | - (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!); + ((SUBSTRING | FORMAT) LPAREN! anyArg (COMMA! anyArg)? RPAREN!); fiveArgString : GET_DELIMITED_FIELD LPAREN! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg)?)?)?)? RPAREN!; // functions that return Booleans @@ -93,13 +93,13 @@ multiArgBool : (IN) LPAREN! anyArg (COMMA! anyArg)* RPAREN!; // functions that return Numbers (whole or decimal) zeroArgNum : (LENGTH | TO_NUMBER | TO_DECIMAL | COUNT) LPAREN! RPAREN!; oneArgNum : ((INDEX_OF | LAST_INDEX_OF) LPAREN! anyArg RPAREN!) | - (TO_DATE LPAREN! anyArg? RPAREN!) | ((MOD | PLUS | MINUS | MULTIPLY | DIVIDE) LPAREN! anyArg RPAREN!); oneOrTwoArgNum : MATH LPAREN! anyArg (COMMA! anyArg)? RPAREN!; +zeroOrOneOrTwoArgNum : TO_DATE LPAREN! anyArg? (COMMA! anyArg)? RPAREN!; stringFunctionRef : zeroArgString | oneArgString | twoArgString | fiveArgString; booleanFunctionRef : zeroArgBool | oneArgBool | multiArgBool; -numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum; +numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum | zeroOrOneOrTwoArgNum; anyArg : WHOLE_NUMBER | DECIMAL | numberFunctionRef | STRING_LITERAL | zeroArgString | oneArgString | twoArgString | fiveArgString | booleanLiteral | zeroArgBool | oneArgBool | multiArgBool | expression; stringArg : STRING_LITERAL | zeroArgString | oneArgString | twoArgString | expression; diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java index 18396aac8e..4b1ce59f52 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java @@ -1225,8 +1225,10 @@ public class Query { case TO_DATE: { if (argEvaluators.isEmpty()) { return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate"); - } else if (subjectEvaluator.getResultType() == ResultType.STRING) { - return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0))), "toDate"); + } else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 1) { + return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), null), "toDate"); + } else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 2) { + return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "toDate"); } else { return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate"); } @@ -1310,7 +1312,13 @@ public class Query { toStringEvaluator(argEvaluators.get(0), "first argument to lastIndexOf")), "lastIndexOf"); } case FORMAT: { - return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format")), "format"); + if(argEvaluators.size() == 1) { + return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format"), null), "format"); + } else if (argEvaluators.size() == 2) { + return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "format"); + } else { + throw new AttributeExpressionLanguageParsingException("format() function takes 1 or 2 arguments"); + } } case OR: { return addToken(new OrEvaluator(toBooleanEvaluator(subjectEvaluator), toBooleanEvaluator(argEvaluators.get(0))), "or"); diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java index 717cbd5f3f..00ce4308f9 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java @@ -20,6 +20,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; import java.util.Map; +import java.util.TimeZone; import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; @@ -31,10 +32,12 @@ public class FormatEvaluator extends StringEvaluator { private final DateEvaluator subject; private final Evaluator format; + private final Evaluator timeZone; - public FormatEvaluator(final DateEvaluator subject, final Evaluator format) { + public FormatEvaluator(final DateEvaluator subject, final Evaluator format, final Evaluator timeZone) { this.subject = subject; this.format = format; + this.timeZone = timeZone; } @Override @@ -50,7 +53,17 @@ public class FormatEvaluator extends StringEvaluator { return null; } - return new StringQueryResult(new SimpleDateFormat(format, Locale.US).format(subjectValue)); + final SimpleDateFormat sdf = new SimpleDateFormat(format, Locale.US); + + if(timeZone != null) { + final QueryResult tzResult = timeZone.evaluate(attributes); + final String tz = tzResult.getValue(); + if(tz != null && TimeZone.getTimeZone(tz) != null) { + sdf.setTimeZone(TimeZone.getTimeZone(tz)); + } + } + + return new StringQueryResult(sdf.format(subjectValue)); } @Override diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java index 590176e096..387777a340 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java @@ -21,6 +21,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; import java.util.Map; +import java.util.TimeZone; import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult; @@ -32,10 +33,12 @@ public class StringToDateEvaluator extends DateEvaluator { private final Evaluator subject; private final Evaluator format; + private final Evaluator timeZone; - public StringToDateEvaluator(final Evaluator subject, final Evaluator format) { + public StringToDateEvaluator(final Evaluator subject, final Evaluator format, final Evaluator timeZone) { this.subject = subject; this.format = format; + this.timeZone = timeZone; } @Override @@ -46,8 +49,18 @@ public class StringToDateEvaluator extends DateEvaluator { return new DateQueryResult(null); } + final SimpleDateFormat sdf = new SimpleDateFormat(formatValue, Locale.US); + + if(timeZone != null) { + final QueryResult tzResult = timeZone.evaluate(attributes); + final String tz = tzResult.getValue(); + if(tz != null && TimeZone.getTimeZone(tz) != null) { + sdf.setTimeZone(TimeZone.getTimeZone(tz)); + } + } + try { - return new DateQueryResult(new SimpleDateFormat(formatValue, Locale.US).parse(subjectValue)); + return new DateQueryResult(sdf.parse(subjectValue)); } catch (final ParseException e) { throw new IllegalAttributeException("Cannot parse attribute value as a date; date format: " + formatValue + "; attribute value: " + subjectValue); diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java index 2dad9cae7a..03de38f987 100644 --- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java +++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java @@ -193,9 +193,8 @@ public class TestQuery { } @Test - @Ignore("Depends on TimeZone") public void testDateToNumber() { - final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):toNumber()}"); + final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS', 'America/New_York'):toNumber()}"); final Map attributes = new HashMap<>(); attributes.put("dateTime", "2013/11/18 10:22:27.678"); @@ -1331,6 +1330,15 @@ public class TestQuery { verifyEquals("${blue:toDate('yyyyMMddHHmmss'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\")}", attributes, "2013/09/17 16:26:43.000Z"); } + @Test + public void testDateFormatConversionWithTimeZone() { + final Map attributes = new HashMap<>(); + attributes.put("blue", "20130917162643"); + verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'GMT')}", attributes, "2013/09/17 16:26:43.000Z"); + verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'Europe/Paris')}", attributes, "2013/09/17 18:26:43.000Z"); + verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'America/Los_Angeles')}", attributes, "2013/09/17 09:26:43.000Z"); + } + @Test public void testNot() { verifyEquals("${ab:notNull():not()}", new HashMap(), true); diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc index 2b629e56a6..7313ce8a9e 100644 --- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc +++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc @@ -1754,13 +1754,15 @@ In order to run the correct method, the parameter types must be correct. The Exp *Description*: [.description]#Formats a number as a date/time according to the format specified by the argument. The argument must be a String that is a valid Java SimpleDateFormat format. The Subject is expected to be a Number that - represents the number of milliseconds since Midnight GMT on January 1, 1970.# + represents the number of milliseconds since Midnight GMT on January 1, 1970. The number will be evaluated using the local + time zone unless specified in the second optional argument.# *Subject Type*: [.subject]#Number# *Arguments*: - [.argName]#_format_# : [.argDesc]#The format to use in the Java SimpleDateFormat syntax# + - [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use (in the Java TimeZone syntax)# *Return Type*: [.returnType]#String# @@ -1770,10 +1772,12 @@ In order to run the correct method, the parameter types must be correct. The Exp .format Examples |============================================================================ | Expression | Value -| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` | `2014/12/31 15:36:03.264Z` -| `${time:format("yyyy/MM/dd")}` | `2014/12/31` -| `${time:format("HH:mm:ss.SSS'Z'")}` | `15:36:03.264Z` -| `${time:format("2014")}` | `2014` +| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` | `2014/12/31 20:36:03.264Z` +| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "America/Los_Angeles")}` | `2014/12/31 12:36:03.264Z` +| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "Asia/Tokyo")}` | `2015/01/01 05:36:03.264Z` +| `${time:format("yyyy/MM/dd", "GMT")}` | `2014/12/31` +| `${time:format("HH:mm:ss.SSS'Z'", "GMT")}` | `20:36:03.264Z` +| `${time:format("yyyy", "GMT")}` | `2014` |============================================================================ @@ -1785,20 +1789,21 @@ In order to run the correct method, the parameter types must be correct. The Exp *Description*: [.description]#Converts a String into a Date data type, based on the format specified by the argument. The argument must be a String that is a valid Java SimpleDateFormat syntax. The Subject is expected to be a String that is formatted - according the argument.# + according the argument. The date will be evaluated using the local time zone unless specified in the second optional argument.# *Subject Type*: [.subject]#String# *Arguments*: - [.argName]#_format_# : [.argDesc]#The current format to use when parsing the Subject, in the Java SimpleDateFormat syntax.# + - [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use when parsing the Subject, in the Java TimeZone syntax.# *Return Type*: [.returnType]#Date# *Examples*: If the attribute "year" has the value "2014" and the attribute "time" has the value "2014/12/31 15:36:03.264Z", - then the Expression `${year:toDate('yyyy')}` will return a Date data type with a value representing Midnight GMT on - January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` will result in a Date data type for + then the Expression `${year:toDate('yyyy', 'GMT')}` will return a Date data type with a value representing Midnight GMT on + January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` will result in a Date data type for 15:36:03.264 GMT on December 31, 2014. Often, this function is used in conjunction with the <> function to change the format of a date/time. For example,