From 49256d9d83572bf7a0d7b16aa02ad43049bf2585 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Mon, 5 Jan 2015 15:40:32 -0500 Subject: [PATCH 1/2] NIFI-22: Added 'count', 'join', 'isEmpty', and 'replaceEmpty' functions --- .../language/antlr/AttributeExpressionLexer.g | 5 +- .../antlr/AttributeExpressionParser.g | 8 +- .../attribute/expression/language/Query.java | 127 ++++++++---------- .../functions/IsEmptyEvaluator.java | 43 ++++++ .../functions/ReplaceEmptyEvaluator.java | 50 +++++++ .../evaluation/reduce/CountEvaluator.java | 56 ++++++++ .../evaluation/reduce/JoinEvaluator.java | 59 ++++++++ .../evaluation/reduce/ReduceEvaluator.java | 23 ++++ .../selection/MappingEvaluator.java | 61 +++++++++ .../expression/language/TestQuery.java | 66 ++++++++- 10 files changed, 422 insertions(+), 76 deletions(-) create mode 100644 commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java create mode 100644 commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java create mode 100644 commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java create mode 100644 commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java create mode 100644 commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java create mode 100644 commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java diff --git a/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g b/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g index 8cb6847f80..10394b9902 100644 --- a/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g +++ b/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g @@ -109,11 +109,13 @@ TO_STRING : 'toString'; LENGTH : 'length'; TRIM : 'trim'; IS_NULL : 'isNull'; +IS_EMPTY : 'isEmpty'; NOT_NULL : 'notNull'; TO_NUMBER : 'toNumber'; URL_ENCODE : 'urlEncode'; URL_DECODE : 'urlDecode'; NOT : 'not'; +COUNT : 'count'; // 1 arg functions SUBSTRING_AFTER : 'substringAfter'; @@ -128,6 +130,7 @@ APPEND : 'append'; INDEX_OF : 'indexOf'; LAST_INDEX_OF : 'lastIndexOf'; REPLACE_NULL : 'replaceNull'; +REPLACE_EMPTY : 'replaceEmpty'; FIND : 'find'; // regex MATCHES : 'matches'; // regex EQUALS : 'equals'; @@ -146,7 +149,7 @@ DIVIDE : 'divide'; TO_RADIX : 'toRadix'; OR : 'or'; AND : 'and'; - +JOIN : 'join'; // 2 arg functions SUBSTRING : 'substring'; diff --git a/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g b/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g index cf10fc044e..f6a87ddcc8 100644 --- a/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g +++ b/commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g @@ -74,15 +74,15 @@ tokens { // functions that return Strings zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE) LPAREN! RPAREN!; -oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | - PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS) LPAREN! anyArg RPAREN!) | +oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY | + PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN) LPAREN! anyArg RPAREN!) | (TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!); twoArgString : ((REPLACE | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) | (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!); // functions that return Booleans -zeroArgBool : (IS_NULL | NOT_NULL | NOT) LPAREN! RPAREN!; +zeroArgBool : (IS_NULL | NOT_NULL | IS_EMPTY | NOT) LPAREN! RPAREN!; oneArgBool : ((FIND | MATCHES | EQUALS_IGNORE_CASE) LPAREN! anyArg RPAREN!) | (GREATER_THAN | LESS_THAN | GREATER_THAN_OR_EQUAL | LESS_THAN_OR_EQUAL) LPAREN! anyArg RPAREN! | (EQUALS) LPAREN! anyArg RPAREN! | @@ -90,7 +90,7 @@ oneArgBool : ((FIND | MATCHES | EQUALS_IGNORE_CASE) LPAREN! anyArg RPAREN!) | // functions that return Numbers -zeroArgNum : (LENGTH | TO_NUMBER) LPAREN! RPAREN!; +zeroArgNum : (LENGTH | TO_NUMBER | COUNT) LPAREN! RPAREN!; oneArgNum : ((INDEX_OF | LAST_INDEX_OF) LPAREN! anyArg RPAREN!) | (TO_DATE LPAREN! anyArg? RPAREN!) | ((MOD | PLUS | MINUS | MULTIPLY | DIVIDE) LPAREN! anyArg RPAREN!); diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java index 93ab7ada64..9a5fe0a746 100644 --- a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java @@ -16,69 +16,7 @@ */ package org.apache.nifi.attribute.expression.language; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_MATCHING_ATTRIBUTES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.AND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_ATTRIBUTE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_DELINEATED_VALUE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_MATCHING_ATTRIBUTE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.APPEND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTRIBUTE_REFERENCE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTR_NAME; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.CONTAINS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.DIVIDE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ENDS_WITH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS_IGNORE_CASE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EXPRESSION; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FALSE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FIND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FORMAT; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN_OR_EQUAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.HOSTNAME; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.INDEX_OF; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IP; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IS_NULL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LAST_INDEX_OF; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LENGTH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN_OR_EQUAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATCHES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MINUS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MOD; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTIPLY; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTI_ATTRIBUTE_REFERENCE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NEXT_INT; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT_NULL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOW; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NUMBER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.OR; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PLUS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PREPEND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_ALL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_NULL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STARTS_WITH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STRING_LITERAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER_LAST; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE_LAST; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_DATE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_LOWER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_NUMBER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_RADIX; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_STRING; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_UPPER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRIM; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRUE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_DECODE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_ENCODE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.*; import java.net.UnknownHostException; import java.util.ArrayList; @@ -118,6 +56,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.Greate import org.apache.nifi.attribute.expression.language.evaluation.functions.HostnameEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.IPEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.IndexOfEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.functions.IsEmptyEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.IsNullEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.LastIndexOfEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.LengthEvaluator; @@ -136,6 +75,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.OrEval import org.apache.nifi.attribute.expression.language.evaluation.functions.PlusEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.PrependEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceAllEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEmptyEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceNullEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.StartsWithEvaluator; @@ -157,6 +97,9 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.UuidEv import org.apache.nifi.attribute.expression.language.evaluation.literals.BooleanLiteralEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.literals.NumberLiteralEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.literals.StringLiteralEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.reduce.CountEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.reduce.JoinEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.reduce.ReduceEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.selection.AllAttributesEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.selection.AnyAttributeEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.selection.DelineatedAttributeEvaluator; @@ -169,7 +112,6 @@ import org.apache.nifi.expression.AttributeExpression.ResultType; import org.apache.nifi.expression.AttributeValueDecorator; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.exception.ProcessException; - import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.CharStream; import org.antlr.runtime.CommonTokenStream; @@ -549,7 +491,8 @@ public class Query { final Tree tree = ast.getChild(0); // ensure that we are able to build the evaluators, so that we validate syntax - buildEvaluator(tree); + final Evaluator evaluator = buildEvaluator(tree); + verifyMappingEvaluatorReduced(evaluator); return tree; } catch (final AttributeExpressionLanguageParsingException e) { throw e; @@ -605,7 +548,10 @@ public class Query { final Tree ast = (Tree) parser.query().getTree(); final Tree tree = ast.getChild(0); - return new Query(query, tree, buildEvaluator(tree)); + final Evaluator evaluator = buildEvaluator(tree); + verifyMappingEvaluatorReduced(evaluator); + + return new Query(query, tree, evaluator); } catch (final AttributeExpressionLanguageParsingException e) { throw e; } catch (final Exception e) { @@ -613,6 +559,33 @@ public class Query { } } + + private static void verifyMappingEvaluatorReduced(final Evaluator evaluator) { + // if the result type of the evaluator is BOOLEAN, then it will always + // be reduced when evaluator. + final ResultType resultType = evaluator.getResultType(); + if (resultType == ResultType.BOOLEAN) { + return; + } + + final Evaluator rootEvaluator = getRootSubjectEvaluator(evaluator); + if (rootEvaluator != null && rootEvaluator instanceof MultiAttributeEvaluator) { + final MultiAttributeEvaluator multiAttrEval = (MultiAttributeEvaluator) rootEvaluator; + switch (multiAttrEval.getEvaluationType()) { + case ALL_ATTRIBUTES: + case ALL_MATCHING_ATTRIBUTES: + case ALL_DELINEATED_VALUES: { + if (!(evaluator instanceof ReduceEvaluator)) { + throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function"); + } + break; + } + default: + throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function"); + } + } + } + private static CommonTokenStream createTokenStream(final String expression) throws AttributeExpressionLanguageParsingException { final CharStream input = new ANTLRStringStream(expression); final AttributeExpressionLexer lexer = new AttributeExpressionLexer(input); @@ -836,10 +809,6 @@ public class Query { final Evaluator rootEvaluator = getRootSubjectEvaluator(evaluator); if (rootEvaluator != null) { if (rootEvaluator instanceof MultiAttributeEvaluator) { - if (evaluator.getResultType() != ResultType.BOOLEAN) { - throw new AttributeExpressionLanguageParsingException("Found Multi-Attribute function but return type is " + evaluator.getResultType() + ", not " + ResultType.BOOLEAN + ", for query: " + tree.getText()); - } - final MultiAttributeEvaluator multiAttrEval = (MultiAttributeEvaluator) rootEvaluator; switch (multiAttrEval.getEvaluationType()) { @@ -926,6 +895,8 @@ public class Query { return (NumberEvaluator) evaluator; case STRING: return new NumberCastEvaluator((StringEvaluator) evaluator); + case DATE: + return new DateToNumberEvaluator((DateEvaluator) evaluator); default: throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " + evaluator.getResultType() + " to " + ResultType.NUMBER + (location == null ? "" : " at location [" + location + "]")); @@ -995,6 +966,10 @@ public class Query { return new ReplaceNullEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argument to replaceNull")); } + case REPLACE_EMPTY: { + verifyArgCount(argEvaluators, 1, "replaceEmtpy"); + return new ReplaceEmptyEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0), "first argumen to replaceEmpty")); + } case REPLACE: { verifyArgCount(argEvaluators, 2, "replace"); return new ReplaceEvaluator(toStringEvaluator(subjectEvaluator), @@ -1030,10 +1005,22 @@ public class Query { throw new AttributeExpressionLanguageParsingException("substring() function can take either 1 or 2 arguments but cannot take " + numArgs + " arguments"); } } + case JOIN: { + verifyArgCount(argEvaluators, 1, "join"); + return new JoinEvaluator(toStringEvaluator(subjectEvaluator), toStringEvaluator(argEvaluators.get(0))); + } + case COUNT: { + verifyArgCount(argEvaluators, 0, "count"); + return new CountEvaluator(subjectEvaluator); + } case IS_NULL: { verifyArgCount(argEvaluators, 0, "isNull"); return new IsNullEvaluator(toStringEvaluator(subjectEvaluator)); } + case IS_EMPTY: { + verifyArgCount(argEvaluators, 0, "isNull"); + return new IsEmptyEvaluator(toStringEvaluator(subjectEvaluator)); + } case NOT_NULL: { verifyArgCount(argEvaluators, 0, "notNull"); return new NotNullEvaluator(toStringEvaluator(subjectEvaluator)); diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java new file mode 100644 index 0000000000..c5e3c2188f --- /dev/null +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/IsEmptyEvaluator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.functions; + +import java.util.Map; + +import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; + +public class IsEmptyEvaluator extends BooleanEvaluator { + private final Evaluator subjectEvaluator; + + public IsEmptyEvaluator(final Evaluator subjectEvaluator) { + this.subjectEvaluator = subjectEvaluator; + } + + @Override + public QueryResult evaluate(final Map attributes) { + final Object subjectValue = subjectEvaluator.evaluate(attributes).getValue(); + return new BooleanQueryResult(subjectValue == null || subjectValue.toString().trim().isEmpty()); + } + + @Override + public Evaluator getSubjectEvaluator() { + return subjectEvaluator; + } +} diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java new file mode 100644 index 0000000000..e5c40d229b --- /dev/null +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceEmptyEvaluator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.functions; + +import java.util.Map; + +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator; + +public class ReplaceEmptyEvaluator extends StringEvaluator { + private final StringEvaluator subjectEvaluator; + private final StringEvaluator replacementEvaluator; + + public ReplaceEmptyEvaluator(final StringEvaluator subjectEvaluator, final StringEvaluator replacementEvaluator) { + this.subjectEvaluator = subjectEvaluator; + this.replacementEvaluator = replacementEvaluator; + } + + @Override + public QueryResult evaluate(final Map attributes) { + final QueryResult subjectResult = subjectEvaluator.evaluate(attributes); + final String subjectValue = subjectResult.getValue(); + final boolean isEmpty = subjectValue == null || subjectValue.toString().trim().isEmpty(); + if ( isEmpty ) { + return replacementEvaluator.evaluate(attributes); + } else { + return subjectResult; + } + } + + @Override + public Evaluator getSubjectEvaluator() { + return subjectEvaluator; + } +} diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java new file mode 100644 index 0000000000..f2af268813 --- /dev/null +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/CountEvaluator.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.reduce; + +import java.util.Map; + +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; +import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.expression.AttributeExpression.ResultType; + +public class CountEvaluator extends NumberEvaluator implements ReduceEvaluator { + + private final Evaluator subjectEvaluator; + private long count = 0L; + + public CountEvaluator(final Evaluator subjectEvaluator) { + this.subjectEvaluator = subjectEvaluator; + } + + @Override + public QueryResult evaluate(final Map attributes) { + final QueryResult result = subjectEvaluator.evaluate(attributes); + if ( result.getValue() == null ) { + return new NumberQueryResult(count); + } + + if ( result.getResultType() == ResultType.BOOLEAN && ((Boolean) result.getValue()).equals(Boolean.FALSE) ) { + return new NumberQueryResult(count); + } + + count++; + return new NumberQueryResult(count); + } + + @Override + public Evaluator getSubjectEvaluator() { + return subjectEvaluator; + } + +} diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java new file mode 100644 index 0000000000..6fab871887 --- /dev/null +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.reduce; + +import java.util.Map; + +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult; + +public class JoinEvaluator extends StringEvaluator implements ReduceEvaluator { + private final StringEvaluator subjectEvaluator; + private final StringEvaluator delimiterEvaluator; + + private final StringBuilder sb = new StringBuilder(); + private int evalCount = 0; + + public JoinEvaluator(final StringEvaluator subject, final StringEvaluator delimiter) { + this.subjectEvaluator = subject; + this.delimiterEvaluator = delimiter; + } + + @Override + public QueryResult evaluate(final Map attributes) { + final String subject = subjectEvaluator.evaluate(attributes).getValue(); + if ( subject == null ) { + return new StringQueryResult(""); + } + + final String delimiter = delimiterEvaluator.evaluate(attributes).getValue(); + if ( evalCount > 0 ) { + sb.append(delimiter); + } + sb.append(subject); + + evalCount++; + return new StringQueryResult( sb.toString() ); + } + + @Override + public Evaluator getSubjectEvaluator() { + return subjectEvaluator; + } +} diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java new file mode 100644 index 0000000000..12197c0ad3 --- /dev/null +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/ReduceEvaluator.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.reduce; + +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; + +public interface ReduceEvaluator extends Evaluator { + +} diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java new file mode 100644 index 0000000000..61ff2af1a4 --- /dev/null +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.selection; + +import java.util.Map; + +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.reduce.ReduceEvaluator; +import org.apache.nifi.expression.AttributeExpression.ResultType; + +public class MappingEvaluator implements Evaluator { + private final ReduceEvaluator mappingEvaluator; + private final MultiAttributeEvaluator multiAttributeEvaluator; + + public MappingEvaluator(final ReduceEvaluator mappingEvaluator, final MultiAttributeEvaluator multiAttributeEval) { + this.mappingEvaluator = mappingEvaluator; + this.multiAttributeEvaluator = multiAttributeEval; + } + + @Override + public QueryResult evaluate(final Map attributes) { + QueryResult result = mappingEvaluator.evaluate(attributes); + + while ( multiAttributeEvaluator.getEvaluationsRemaining() > 0 ) { + result = mappingEvaluator.evaluate(attributes); + } + + return result; + } + + @Override + public ResultType getResultType() { + return mappingEvaluator.getResultType(); + } + + @Override + public int getEvaluationsRemaining() { + return 0; + } + + @Override + public Evaluator getSubjectEvaluator() { + return mappingEvaluator; + } + +} diff --git a/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java index dd7a7fe36c..824249ba79 100644 --- a/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java +++ b/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java @@ -29,10 +29,10 @@ import java.util.Map; import org.apache.nifi.attribute.expression.language.Query.Range; import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException; import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException; import org.apache.nifi.expression.AttributeExpression.ResultType; import org.apache.nifi.flowfile.FlowFile; - import org.antlr.runtime.tree.Tree; import org.junit.Assert; import org.junit.Ignore; @@ -192,9 +192,11 @@ public class TestQuery { attributes.put("dateTime", "2013/11/18 10:22:27.678"); verifyEquals("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):toNumber():plus(86400000):toDate():format('yyyy/MM/dd HH:mm:ss.SSS')}", attributes, "2013/11/19 10:22:27.678"); + verifyEquals("${dateTime:toDate('yyyy/MM/dd HH:mm:ss.SSS'):plus(86400000):format('yyyy/MM/dd HH:mm:ss.SSS')}", attributes, "2013/11/19 10:22:27.678"); } @Test + @Ignore("Requires specific locale") public void implicitDateConversion() { final Date date = new Date(); final Query query = Query.compile("${dateTime:format('yyyy/MM/dd HH:mm:ss.SSS')}"); @@ -229,6 +231,68 @@ public class TestQuery { assertEquals("true", Query.evaluateExpressions("${x:equals(\"${a}\")}", attributes, null)); } + @Test + public void testJoin() { + final Map attributes = new HashMap<>(); + attributes.put("a.a", "a"); + attributes.put("b.b", "b"); + attributes.put("c.c", "c"); + verifyEquals("${allAttributes( 'a.a', 'a.b', 'a.c' ):join(', ')}", attributes, "a, b, c"); + verifyEquals("${x:join(', ')}", attributes, ""); + verifyEquals("${a.a:join(', ')}", attributes, "a"); + verifyEquals("${allAttributes( 'x', 'y' ):join(',')}", attributes, ","); + } + + @Test(expected=AttributeExpressionLanguageException.class) + public void testCannotCombineWithNonReducingFunction() { + Query.compileTree("${allAttributes( 'a.1' ):plus(1)}"); + } + + + @Test + public void testIsEmpty() { + final Map attributes = new HashMap<>(); + attributes.put("a", "a"); + attributes.put("b", ""); + attributes.put("c", " \n"); + + verifyEquals("${a:isEmpty()}", attributes, false); + verifyEquals("${b:isEmpty()}", attributes, true); + verifyEquals("${c:isEmpty()}", attributes, true); + verifyEquals("${d:isEmpty()}", attributes, true); + } + + + @Test + public void testReplaceEmpty() { + final Map attributes = new HashMap<>(); + attributes.put("a", "a"); + attributes.put("b", ""); + attributes.put("c", " \n"); + + verifyEquals("${a:replaceEmpty('c')}", attributes, "a"); + verifyEquals("${b:replaceEmpty('c')}", attributes, "c"); + verifyEquals("${c:replaceEmpty('c')}", attributes, "c"); + verifyEquals("${d:replaceEmpty('c')}", attributes, "c"); + } + + + + @Test + public void testCount() { + final Map attributes = new HashMap<>(); + attributes.put("a", "a"); + attributes.put("b", ""); + attributes.put("c", " \n"); + attributes.put("n1", "111"); + attributes.put("n2", "222"); + attributes.put("n3", "333333"); + + verifyEquals("${allMatchingAttributes( '.*' ):count()}", attributes, 6L); + verifyEquals("${allMatchingAttributes( '.*' ):length():gt(2):count()}", attributes, 5L); + verifyEquals("${allMatchingAttributes( 'n.*' ):plus(1):count()}", attributes, 3L ); + } + @Test public void testCurlyBracesInQuotes() { From 14cba2a57ec78cd1bb8e664474f0db1e6f2799b9 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 7 Jan 2015 14:16:51 -0500 Subject: [PATCH 2/2] NIFI-22: fixed issues with newly added join and count functions --- .../attribute/expression/language/Query.java | 50 +++++++++++-------- .../evaluation/reduce/JoinEvaluator.java | 4 +- .../selection/MappingEvaluator.java | 2 +- .../expression/language/TestQuery.java | 6 +-- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java index 9a5fe0a746..420a8e21c7 100644 --- a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java @@ -116,6 +116,7 @@ import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.CharStream; import org.antlr.runtime.CommonTokenStream; import org.antlr.runtime.tree.Tree; +import org.apache.nifi.attribute.expression.language.evaluation.selection.MappingEvaluator; /** * Class used for creating and evaluating NiFi Expression Language. Once a Query @@ -515,26 +516,26 @@ public class Query { try { final List substrings = new ArrayList<>(); final Map trees = new HashMap<>(); - + int lastIndex = 0; for (final Range range : ranges) { if (range.getStart() > lastIndex) { substrings.add(query.substring(lastIndex, range.getStart()).replace("$$", "$")); lastIndex = range.getEnd() + 1; } - + final String treeText = query.substring(range.getStart(), range.getEnd() + 1).replace("$$", "$"); substrings.add(treeText); trees.put(treeText, Query.compileTree(treeText)); lastIndex = range.getEnd() + 1; } - + final Range lastRange = ranges.get(ranges.size() - 1); if (lastRange.getEnd() + 1 < query.length()) { final String treeText = query.substring(lastRange.getEnd() + 1).replace("$$", "$"); substrings.add(treeText); } - + return new StandardPreparedQuery(substrings, trees); } catch (final AttributeExpressionLanguageParsingException e) { return new InvalidPreparedQuery(query, e.getMessage()); @@ -550,7 +551,7 @@ public class Query { final Evaluator evaluator = buildEvaluator(tree); verifyMappingEvaluatorReduced(evaluator); - + return new Query(query, tree, evaluator); } catch (final AttributeExpressionLanguageParsingException e) { throw e; @@ -559,7 +560,6 @@ public class Query { } } - private static void verifyMappingEvaluatorReduced(final Evaluator evaluator) { // if the result type of the evaluator is BOOLEAN, then it will always // be reduced when evaluator. @@ -572,20 +572,20 @@ public class Query { if (rootEvaluator != null && rootEvaluator instanceof MultiAttributeEvaluator) { final MultiAttributeEvaluator multiAttrEval = (MultiAttributeEvaluator) rootEvaluator; switch (multiAttrEval.getEvaluationType()) { - case ALL_ATTRIBUTES: - case ALL_MATCHING_ATTRIBUTES: - case ALL_DELINEATED_VALUES: { - if (!(evaluator instanceof ReduceEvaluator)) { - throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function"); + case ALL_ATTRIBUTES: + case ALL_MATCHING_ATTRIBUTES: + case ALL_DELINEATED_VALUES: { + if (!(evaluator instanceof ReduceEvaluator)) { + throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function"); + } + break; } - break; - } - default: - throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function"); + default: + throw new AttributeExpressionLanguageParsingException("Cannot evaluate expression because it attempts to reference multiple attributes but does not use a reducing function"); } } } - + private static CommonTokenStream createTokenStream(final String expression) throws AttributeExpressionLanguageParsingException { final CharStream input = new ANTLRStringStream(expression); final AttributeExpressionLexer lexer = new AttributeExpressionLexer(input); @@ -791,7 +791,7 @@ public class Query { if (tree.getChildCount() == 0) { throw new AttributeExpressionLanguageParsingException("EXPRESSION tree node has no children"); } - + final Evaluator evaluator; if (tree.getChildCount() == 1) { evaluator = buildEvaluator(tree.getChild(0)); @@ -804,7 +804,7 @@ public class Query { // tree from the right-most child going left-ward. evaluator = buildFunctionExpressionEvaluator(tree, 0); } - + Evaluator chosenEvaluator = evaluator; final Evaluator rootEvaluator = getRootSubjectEvaluator(evaluator); if (rootEvaluator != null) { @@ -819,13 +819,21 @@ public class Query { break; case ALL_ATTRIBUTES: case ALL_MATCHING_ATTRIBUTES: - case ALL_DELINEATED_VALUES: - chosenEvaluator = new AllAttributesEvaluator((BooleanEvaluator) evaluator, multiAttrEval); + case ALL_DELINEATED_VALUES: { + final ResultType resultType = evaluator.getResultType(); + if (resultType == ResultType.BOOLEAN) { + chosenEvaluator = new AllAttributesEvaluator((BooleanEvaluator) evaluator, multiAttrEval); + } else if (evaluator instanceof ReduceEvaluator) { + chosenEvaluator = new MappingEvaluator((ReduceEvaluator) evaluator, multiAttrEval); + } else { + throw new AttributeExpressionLanguageException("Cannot evaluate Expression because it attempts to reference multiple attributes but does not use a reducing function"); + } break; + } } } } - + return chosenEvaluator; } diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java index 6fab871887..eefdadad19 100644 --- a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/reduce/JoinEvaluator.java @@ -37,9 +37,9 @@ public class JoinEvaluator extends StringEvaluator implements ReduceEvaluator evaluate(final Map attributes) { - final String subject = subjectEvaluator.evaluate(attributes).getValue(); + String subject = subjectEvaluator.evaluate(attributes).getValue(); if ( subject == null ) { - return new StringQueryResult(""); + subject = ""; } final String delimiter = delimiterEvaluator.evaluate(attributes).getValue(); diff --git a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java index 61ff2af1a4..d872b6e4c2 100644 --- a/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java +++ b/commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/selection/MappingEvaluator.java @@ -55,7 +55,7 @@ public class MappingEvaluator implements Evaluator { @Override public Evaluator getSubjectEvaluator() { - return mappingEvaluator; + return null; } } diff --git a/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java index 824249ba79..ad0065cf38 100644 --- a/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java +++ b/commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java @@ -235,8 +235,8 @@ public class TestQuery { public void testJoin() { final Map attributes = new HashMap<>(); attributes.put("a.a", "a"); - attributes.put("b.b", "b"); - attributes.put("c.c", "c"); + attributes.put("a.b", "b"); + attributes.put("a.c", "c"); verifyEquals("${allAttributes( 'a.a', 'a.b', 'a.c' ):join(', ')}", attributes, "a, b, c"); verifyEquals("${x:join(', ')}", attributes, ""); verifyEquals("${a.a:join(', ')}", attributes, "a"); @@ -282,7 +282,7 @@ public class TestQuery { public void testCount() { final Map attributes = new HashMap<>(); attributes.put("a", "a"); - attributes.put("b", ""); + attributes.put("b", "abc"); attributes.put("c", " \n"); attributes.put("n1", "111"); attributes.put("n2", "222");