NIFI-2908 Added TimeZone to toDate() and format() EL

This closes #1381.

Signed-off-by: Koji Kawamura <ijokarumawak@apache.org>

nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g

@@ -75,10 +75,10 @@ tokens {
 // functions that return Strings
 zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!;
 oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
-				PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
+				PREPEND | APPEND | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
 			   (TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
 twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL | IF_ELSE) LPAREN! anyArg COMMA! anyArg RPAREN!) |
-			   (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
+			   ((SUBSTRING | FORMAT) LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
 fiveArgString : GET_DELIMITED_FIELD LPAREN! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg)?)?)?)? RPAREN!;
 
 // functions that return Booleans
@@ -93,13 +93,13 @@ multiArgBool : (IN) LPAREN! anyArg (COMMA! anyArg)* RPAREN!;
 // functions that return Numbers (whole or decimal)
 zeroArgNum	: (LENGTH | TO_NUMBER | TO_DECIMAL | COUNT) LPAREN! RPAREN!;
 oneArgNum	: ((INDEX_OF | LAST_INDEX_OF) LPAREN! anyArg RPAREN!) |
-			  (TO_DATE LPAREN! anyArg? RPAREN!) |
 			  ((MOD | PLUS | MINUS | MULTIPLY | DIVIDE) LPAREN! anyArg RPAREN!);
 oneOrTwoArgNum : MATH LPAREN! anyArg (COMMA! anyArg)? RPAREN!;
+zeroOrOneOrTwoArgNum : TO_DATE LPAREN! anyArg? (COMMA! anyArg)? RPAREN!;
 
 stringFunctionRef : zeroArgString | oneArgString | twoArgString | fiveArgString;
 booleanFunctionRef : zeroArgBool | oneArgBool | multiArgBool;
-numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum;
+numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum | zeroOrOneOrTwoArgNum;
 
 anyArg : WHOLE_NUMBER | DECIMAL | numberFunctionRef | STRING_LITERAL | zeroArgString | oneArgString | twoArgString | fiveArgString | booleanLiteral | zeroArgBool | oneArgBool | multiArgBool | expression;
 stringArg : STRING_LITERAL | zeroArgString | oneArgString | twoArgString | expression;

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java

@@ -1225,8 +1225,10 @@ public class Query {
             case TO_DATE: {
                 if (argEvaluators.isEmpty()) {
                     return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate");
-                } else if (subjectEvaluator.getResultType() == ResultType.STRING) {
+                } else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 1) {
-                    return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0))), "toDate");
+                    return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), null), "toDate");
+                } else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 2) {
+                    return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "toDate");
                 } else {
                     return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate");
                 }
@@ -1310,7 +1312,13 @@ public class Query {
                     toStringEvaluator(argEvaluators.get(0), "first argument to lastIndexOf")), "lastIndexOf");
             }
             case FORMAT: {
-                return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format")), "format");
+                if(argEvaluators.size() == 1) {
+                    return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format"), null), "format");
+                } else if (argEvaluators.size() == 2) {
+                    return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "format");
+                } else {
+                    throw new AttributeExpressionLanguageParsingException("format() function takes 1 or 2 arguments");
+                }
             }
             case OR: {
                 return addToken(new OrEvaluator(toBooleanEvaluator(subjectEvaluator), toBooleanEvaluator(argEvaluators.get(0))), "or");

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java

@@ -20,6 +20,7 @@ import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.Locale;
 import java.util.Map;
+import java.util.TimeZone;
 
 import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
 import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
@@ -31,10 +32,12 @@ public class FormatEvaluator extends StringEvaluator {
 
     private final DateEvaluator subject;
     private final Evaluator<String> format;
+    private final Evaluator<String> timeZone;
 
-    public FormatEvaluator(final DateEvaluator subject, final Evaluator<String> format) {
+    public FormatEvaluator(final DateEvaluator subject, final Evaluator<String> format, final Evaluator<String> timeZone) {
         this.subject = subject;
         this.format = format;
+        this.timeZone = timeZone;
     }
 
     @Override
@@ -50,7 +53,17 @@ public class FormatEvaluator extends StringEvaluator {
             return null;
         }
 
-        return new StringQueryResult(new SimpleDateFormat(format, Locale.US).format(subjectValue));
+        final SimpleDateFormat sdf = new SimpleDateFormat(format, Locale.US);
+
+        if(timeZone != null) {
+            final QueryResult<String> tzResult = timeZone.evaluate(attributes);
+            final String tz = tzResult.getValue();
+            if(tz != null && TimeZone.getTimeZone(tz) != null) {
+                sdf.setTimeZone(TimeZone.getTimeZone(tz));
+            }
+        }
+
+        return new StringQueryResult(sdf.format(subjectValue));
     }
 
     @Override

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java

@@ -21,6 +21,7 @@ import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.Locale;
 import java.util.Map;
+import java.util.TimeZone;
 
 import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
 import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
@@ -32,10 +33,12 @@ public class StringToDateEvaluator extends DateEvaluator {
 
     private final Evaluator<String> subject;
     private final Evaluator<String> format;
+    private final Evaluator<String> timeZone;
 
-    public StringToDateEvaluator(final Evaluator<String> subject, final Evaluator<String> format) {
+    public StringToDateEvaluator(final Evaluator<String> subject, final Evaluator<String> format, final Evaluator<String> timeZone) {
         this.subject = subject;
         this.format = format;
+        this.timeZone = timeZone;
     }
 
     @Override
@@ -46,8 +49,18 @@ public class StringToDateEvaluator extends DateEvaluator {
             return new DateQueryResult(null);
         }
 
+        final SimpleDateFormat sdf = new SimpleDateFormat(formatValue, Locale.US);
+
+        if(timeZone != null) {
+            final QueryResult<String> tzResult = timeZone.evaluate(attributes);
+            final String tz = tzResult.getValue();
+            if(tz != null && TimeZone.getTimeZone(tz) != null) {
+                sdf.setTimeZone(TimeZone.getTimeZone(tz));
+            }
+        }
+
         try {
-            return new DateQueryResult(new SimpleDateFormat(formatValue, Locale.US).parse(subjectValue));
+            return new DateQueryResult(sdf.parse(subjectValue));
         } catch (final ParseException e) {
             throw new IllegalAttributeException("Cannot parse attribute value as a date; date format: "
                     + formatValue + "; attribute value: " + subjectValue);

nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java

@@ -193,9 +193,8 @@ public class TestQuery {
     }
 
     @Test
-    @Ignore("Depends on TimeZone")
     public void testDateToNumber() {
-        final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):toNumber()}");
+        final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS', 'America/New_York'):toNumber()}");
         final Map<String, String> attributes = new HashMap<>();
         attributes.put("dateTime", "2013/11/18 10:22:27.678");
 
@@ -1331,6 +1330,15 @@ public class TestQuery {
         verifyEquals("${blue:toDate('yyyyMMddHHmmss'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\")}", attributes, "2013/09/17 16:26:43.000Z");
     }
 
+    @Test
+    public void testDateFormatConversionWithTimeZone() {
+        final Map<String, String> attributes = new HashMap<>();
+        attributes.put("blue", "20130917162643");
+        verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'GMT')}", attributes, "2013/09/17 16:26:43.000Z");
+        verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'Europe/Paris')}", attributes, "2013/09/17 18:26:43.000Z");
+        verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'America/Los_Angeles')}", attributes, "2013/09/17 09:26:43.000Z");
+    }
+
     @Test
     public void testNot() {
         verifyEquals("${ab:notNull():not()}", new HashMap<String, String>(), true);

nifi-docs/src/main/asciidoc/expression-language-guide.adoc

@@ -1754,13 +1754,15 @@ In order to run the correct method, the parameter types must be correct. The Exp
 
 *Description*: [.description]#Formats a number as a date/time according to the format specified by the argument. The argument
 	must be a String that is a valid Java SimpleDateFormat format. The Subject is expected to be a Number that
-	represents the number of milliseconds since Midnight GMT on January 1, 1970.#
+	represents the number of milliseconds since Midnight GMT on January 1, 1970. The number will be evaluated using the local 
+	time zone unless specified in the second optional argument.#
 
 *Subject Type*: [.subject]#Number#
 
 *Arguments*:
 
 	- [.argName]#_format_# : [.argDesc]#The format to use in the Java SimpleDateFormat syntax#
+	- [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use (in the Java TimeZone syntax)#
 
 *Return Type*: [.returnType]#String#
 
@@ -1770,10 +1772,12 @@ In order to run the correct method, the parameter types must be correct. The Exp
 .format Examples
 |============================================================================
 | Expression | Value
-| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` | `2014/12/31 15:36:03.264Z`
+| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` | `2014/12/31 20:36:03.264Z`
-| `${time:format("yyyy/MM/dd")}` | `2014/12/31`
+| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "America/Los_Angeles")}` | `2014/12/31 12:36:03.264Z`
-| `${time:format("HH:mm:ss.SSS'Z'")}` | `15:36:03.264Z`
+| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "Asia/Tokyo")}` | `2015/01/01 05:36:03.264Z`
-| `${time:format("2014")}` | `2014`
+| `${time:format("yyyy/MM/dd", "GMT")}` | `2014/12/31`
+| `${time:format("HH:mm:ss.SSS'Z'", "GMT")}` | `20:36:03.264Z`
+| `${time:format("yyyy", "GMT")}` | `2014`
 |============================================================================
 
 
@@ -1785,20 +1789,21 @@ In order to run the correct method, the parameter types must be correct. The Exp
 
 *Description*: [.description]#Converts a String into a Date data type, based on the format specified by the argument. The argument
 	must be a String that is a valid Java SimpleDateFormat syntax. The Subject is expected to be a String that is formatted
-	according the argument.#
+	according the argument. The date will be evaluated using the local time zone unless specified in the second optional argument.#
 
 *Subject Type*: [.subject]#String#
 
 *Arguments*:
 
 		- [.argName]#_format_# : [.argDesc]#The current format to use when parsing the Subject, in the Java SimpleDateFormat syntax.#
+		- [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use when parsing the Subject, in the Java TimeZone syntax.#
 
 
 *Return Type*: [.returnType]#Date#
 
 *Examples*: If the attribute "year" has the value "2014" and the attribute "time" has the value "2014/12/31 15:36:03.264Z",
-	then the Expression `${year:toDate('yyyy')}` will return a Date data type with a value representing Midnight GMT on
+	then the Expression `${year:toDate('yyyy', 'GMT')}` will return a Date data type with a value representing Midnight GMT on
-	January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` will result in a Date data type for
+	January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` will result in a Date data type for
 	15:36:03.264 GMT on December 31, 2014.
 
 Often, this function is used in conjunction with the <<format>> function to change the format of a date/time. For example,