Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-2908 Added TimeZone to toDate() and format() EL 

This closes #1381.

Signed-off-by: Koji Kawamura <ijokarumawak@apache.org>
This commit is contained in:
Pierre Villard 2017-01-03 13:16:40 +01:00 committed by Koji Kawamura
parent ef54a8ec69
commit 3bb876eb83
6 changed files with 68 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
View File

@ -75,10 +75,10 @@ tokens {
// functions that return Strings
zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!;
oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
PREPEND | APPEND | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
(TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL | IF_ELSE) LPAREN! anyArg COMMA! anyArg RPAREN!) |
(SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
((SUBSTRING | FORMAT) LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
fiveArgString : GET_DELIMITED_FIELD LPAREN! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg)?)?)?)? RPAREN!;
// functions that return Booleans
@ -93,13 +93,13 @@ multiArgBool : (IN) LPAREN! anyArg (COMMA! anyArg)* RPAREN!;
// functions that return Numbers (whole or decimal)
zeroArgNum : (LENGTH | TO_NUMBER | TO_DECIMAL | COUNT) LPAREN! RPAREN!;
oneArgNum : ((INDEX_OF | LAST_INDEX_OF) LPAREN! anyArg RPAREN!) |
(TO_DATE LPAREN! anyArg? RPAREN!) |
((MOD | PLUS | MINUS | MULTIPLY | DIVIDE) LPAREN! anyArg RPAREN!);
oneOrTwoArgNum : MATH LPAREN! anyArg (COMMA! anyArg)? RPAREN!;
zeroOrOneOrTwoArgNum : TO_DATE LPAREN! anyArg? (COMMA! anyArg)? RPAREN!;
stringFunctionRef : zeroArgString | oneArgString | twoArgString | fiveArgString;
booleanFunctionRef : zeroArgBool | oneArgBool | multiArgBool;
numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum;
numberFunctionRef : zeroArgNum | oneArgNum | oneOrTwoArgNum | zeroOrOneOrTwoArgNum;
anyArg : WHOLE_NUMBER | DECIMAL | numberFunctionRef | STRING_LITERAL | zeroArgString | oneArgString | twoArgString | fiveArgString | booleanLiteral | zeroArgBool | oneArgBool | multiArgBool | expression;
stringArg : STRING_LITERAL | zeroArgString | oneArgString | twoArgString | expression;

14
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
View File

@ -1225,8 +1225,10 @@ public class Query {
case TO_DATE: {
if (argEvaluators.isEmpty()) {
return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate");
} else if (subjectEvaluator.getResultType() == ResultType.STRING) {
return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0))), "toDate");
} else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 1) {
return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), null), "toDate");
} else if (subjectEvaluator.getResultType() == ResultType.STRING && argEvaluators.size() == 2) {
return addToken(new StringToDateEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "toDate");
} else {
return addToken(new NumberToDateEvaluator(toWholeNumberEvaluator(subjectEvaluator)), "toDate");
}
@ -1310,7 +1312,13 @@ public class Query {
toStringEvaluator(argEvaluators.get(0), "first argument to lastIndexOf")), "lastIndexOf");
}
case FORMAT: {
return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format")), "format");
if(argEvaluators.size() == 1) {
return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument of format"), null), "format");
} else if (argEvaluators.size() == 2) {
return addToken(new FormatEvaluator(toDateEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0)), toStringEvaluator(argEvaluators.get(1))), "format");
} else {
throw new AttributeExpressionLanguageParsingException("format() function takes 1 or 2 arguments");
}
}
case OR: {
return addToken(new OrEvaluator(toBooleanEvaluator(subjectEvaluator), toBooleanEvaluator(argEvaluators.get(0))), "or");

17
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FormatEvaluator.java
View File

@ -20,6 +20,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
@ -31,10 +32,12 @@ public class FormatEvaluator extends StringEvaluator {
private final DateEvaluator subject;
private final Evaluator<String> format;
private final Evaluator<String> timeZone;
public FormatEvaluator(final DateEvaluator subject, final Evaluator<String> format) {
public FormatEvaluator(final DateEvaluator subject, final Evaluator<String> format, final Evaluator<String> timeZone) {
this.subject = subject;
this.format = format;
this.timeZone = timeZone;
}
@Override
@ -50,7 +53,17 @@ public class FormatEvaluator extends StringEvaluator {
return null;
}
return new StringQueryResult(new SimpleDateFormat(format, Locale.US).format(subjectValue));
final SimpleDateFormat sdf = new SimpleDateFormat(format, Locale.US);
if(timeZone != null) {
final QueryResult<String> tzResult = timeZone.evaluate(attributes);
final String tz = tzResult.getValue();
if(tz != null && TimeZone.getTimeZone(tz) != null) {
sdf.setTimeZone(TimeZone.getTimeZone(tz));
}
}
return new StringQueryResult(sdf.format(subjectValue));
}
@Override

17
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/StringToDateEvaluator.java
View File

@ -21,6 +21,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.DateQueryResult;
@ -32,10 +33,12 @@ public class StringToDateEvaluator extends DateEvaluator {
private final Evaluator<String> subject;
private final Evaluator<String> format;
private final Evaluator<String> timeZone;
public StringToDateEvaluator(final Evaluator<String> subject, final Evaluator<String> format) {
public StringToDateEvaluator(final Evaluator<String> subject, final Evaluator<String> format, final Evaluator<String> timeZone) {
this.subject = subject;
this.format = format;
this.timeZone = timeZone;
}
@Override
@ -46,8 +49,18 @@ public class StringToDateEvaluator extends DateEvaluator {
return new DateQueryResult(null);
}
final SimpleDateFormat sdf = new SimpleDateFormat(formatValue, Locale.US);
if(timeZone != null) {
final QueryResult<String> tzResult = timeZone.evaluate(attributes);
final String tz = tzResult.getValue();
if(tz != null && TimeZone.getTimeZone(tz) != null) {
sdf.setTimeZone(TimeZone.getTimeZone(tz));
}
}
try {
return new DateQueryResult(new SimpleDateFormat(formatValue, Locale.US).parse(subjectValue));
return new DateQueryResult(sdf.parse(subjectValue));
} catch (final ParseException e) {
throw new IllegalAttributeException("Cannot parse attribute value as a date; date format: "
+ formatValue + "; attribute value: " + subjectValue);

12
nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
View File

@ -193,9 +193,8 @@ public class TestQuery {
}
@Test
@Ignore("Depends on TimeZone")
public void testDateToNumber() {
final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):toNumber()}");
final Query query = Query.compile("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS', 'America/New_York'):toNumber()}");
final Map<String, String> attributes = new HashMap<>();
attributes.put("dateTime", "2013/11/18 10:22:27.678");
@ -1331,6 +1330,15 @@ public class TestQuery {
verifyEquals("${blue:toDate('yyyyMMddHHmmss'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\")}", attributes, "2013/09/17 16:26:43.000Z");
}
@Test
public void testDateFormatConversionWithTimeZone() {
final Map<String, String> attributes = new HashMap<>();
attributes.put("blue", "20130917162643");
verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'GMT')}", attributes, "2013/09/17 16:26:43.000Z");
verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'Europe/Paris')}", attributes, "2013/09/17 18:26:43.000Z");
verifyEquals("${blue:toDate('yyyyMMddHHmmss', 'GMT'):format(\"yyyy/MM/dd HH:mm:ss.SSS'Z'\", 'America/Los_Angeles')}", attributes, "2013/09/17 09:26:43.000Z");
}
@Test
public void testNot() {
verifyEquals("${ab:notNull():not()}", new HashMap<String, String>(), true);

21
nifi-docs/src/main/asciidoc/expression-language-guide.adoc
View File

@ -1754,13 +1754,15 @@ In order to run the correct method, the parameter types must be correct. The Exp
*Description*: [.description]#Formats a number as a date/time according to the format specified by the argument. The argument
must be a String that is a valid Java SimpleDateFormat format. The Subject is expected to be a Number that
represents the number of milliseconds since Midnight GMT on January 1, 1970.#
represents the number of milliseconds since Midnight GMT on January 1, 1970. The number will be evaluated using the local
time zone unless specified in the second optional argument.#
*Subject Type*: [.subject]#Number#
*Arguments*:
- [.argName]#_format_# : [.argDesc]#The format to use in the Java SimpleDateFormat syntax#
- [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use (in the Java TimeZone syntax)#
*Return Type*: [.returnType]#String#
@ -1770,10 +1772,12 @@ In order to run the correct method, the parameter types must be correct. The Exp
.format Examples
|============================================================================
| Expression | Value
| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` | `2014/12/31 15:36:03.264Z`
| `${time:format("yyyy/MM/dd")}` | `2014/12/31`
| `${time:format("HH:mm:ss.SSS'Z'")}` | `15:36:03.264Z`
| `${time:format("2014")}` | `2014`
| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` | `2014/12/31 20:36:03.264Z`
| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "America/Los_Angeles")}` | `2014/12/31 12:36:03.264Z`
| `${time:format("yyyy/MM/dd HH:mm:ss.SSS'Z'", "Asia/Tokyo")}` | `2015/01/01 05:36:03.264Z`
| `${time:format("yyyy/MM/dd", "GMT")}` | `2014/12/31`
| `${time:format("HH:mm:ss.SSS'Z'", "GMT")}` | `20:36:03.264Z`
| `${time:format("yyyy", "GMT")}` | `2014`
|============================================================================
@ -1785,20 +1789,21 @@ In order to run the correct method, the parameter types must be correct. The Exp
*Description*: [.description]#Converts a String into a Date data type, based on the format specified by the argument. The argument
must be a String that is a valid Java SimpleDateFormat syntax. The Subject is expected to be a String that is formatted
according the argument.#
according the argument. The date will be evaluated using the local time zone unless specified in the second optional argument.#
*Subject Type*: [.subject]#String#
*Arguments*:
- [.argName]#_format_# : [.argDesc]#The current format to use when parsing the Subject, in the Java SimpleDateFormat syntax.#
- [.argName]#_time zone_# : [.argDesc]#Optional argument that specifies the time zone to use when parsing the Subject, in the Java TimeZone syntax.#
*Return Type*: [.returnType]#Date#
*Examples*: If the attribute "year" has the value "2014" and the attribute "time" has the value "2014/12/31 15:36:03.264Z",
then the Expression `${year:toDate('yyyy')}` will return a Date data type with a value representing Midnight GMT on
January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'")}` will result in a Date data type for
then the Expression `${year:toDate('yyyy', 'GMT')}` will return a Date data type with a value representing Midnight GMT on
January 1, 2014. The Expression `${time:toDate("yyyy/MM/dd HH:mm:ss.SSS'Z'", "GMT")}` will result in a Date data type for
15:36:03.264 GMT on December 31, 2014.
Often, this function is used in conjunction with the <<format>> function to change the format of a date/time. For example,