From 812731497553404161eb51100ba34f5121fffad9 Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Tue, 24 May 2016 19:05:34 -0700 Subject: [PATCH] NIFI-1919 Added replaceFirst() expression language method which accepts literal or pattern for replacement. Reverted whitespace changes. (+8 squashed commits) Squashed commits: [329755c] NIFI-1919 Reverted import re-organization from IDE. [cf73c2f] NIFI-1919 Updated expression language guide. [d9a1455] NIFI-1919 Reverted changes to ReplaceEvaluator. Added ReplaceFirstEvaluator. Added replace first logic to Query buildFunctionEvaluator. Added unit tests. [e2eb880] NIFI-1919 Added replaceFirst to AttributeExpression lexer and parser grammar definitions. [11fe913] NIFI-1919 Ignored demonstrative test for replaceAll as it behaves as expected. [af97be1] NIFI-1919 Changed ReplaceEvaluator to use String#replaceFirst which interprets regex instead of compiling as literal. Demonstrative unit test now passes but two existing unit tests fail. I am not sure these tests are correct. [f24f17b] NIFI-1919 Added working unit test to illustrate fix. [8a0d43b] NIFI-1919 Added Groovy unit test to demonstrate issue. Added DelegatingMetaClass code to record it (test not complete). This closes #474. Signed-off-by: Andy LoPresto --- .../language/antlr/AttributeExpressionLexer.g | 1 + .../antlr/AttributeExpressionParser.g | 2 +- .../attribute/expression/language/Query.java | 12 +- .../functions/ReplaceFirstEvaluator.java | 54 +++++ .../language/QueryGroovyTest.groovy | 212 ++++++++++++++++++ .../asciidoc/expression-language-guide.adoc | 38 +++- 6 files changed, 313 insertions(+), 6 deletions(-) create mode 100644 nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceFirstEvaluator.java create mode 100644 nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g index 0243a0bcca..1b3c345817 100644 --- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g +++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g @@ -156,6 +156,7 @@ TO_LITERAL : 'literal'; // 2 arg functions SUBSTRING : 'substring'; REPLACE : 'replace'; +REPLACE_FIRST : 'replaceFirst'; REPLACE_ALL : 'replaceAll'; // 4 arg functions diff --git a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g index dba346c821..5e0c4932d2 100644 --- a/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g +++ b/nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g @@ -77,7 +77,7 @@ zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECOD oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY | PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN) LPAREN! anyArg RPAREN!) | (TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!); -twoArgString : ((REPLACE | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) | +twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) | (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!); fiveArgString : GET_DELIMITED_FIELD LPAREN! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg)?)?)?)? RPAREN!; diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java index c9ccfcb871..4476097754 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java @@ -79,6 +79,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.Random import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceAllEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEmptyEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceFirstEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceNullEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.StartsWithEvaluator; import org.apache.nifi.attribute.expression.language.evaluation.functions.StringToDateEvaluator; @@ -172,6 +173,7 @@ import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpre import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.RANDOM; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_ALL; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_EMPTY; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_FIRST; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_NULL; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STARTS_WITH; import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STRING_LITERAL; @@ -427,8 +429,8 @@ public class Query { } static Map createExpressionMap(final FlowFile flowFile, final Map additionalAttributes) { - final Map attributeMap = flowFile == null ? Collections. emptyMap() : flowFile.getAttributes(); - final Map additionalOrEmpty = additionalAttributes == null ? Collections. emptyMap() : additionalAttributes; + final Map attributeMap = flowFile == null ? Collections.emptyMap() : flowFile.getAttributes(); + final Map additionalOrEmpty = additionalAttributes == null ? Collections.emptyMap() : additionalAttributes; final Map envMap = System.getenv(); final Map sysProps = System.getProperties(); @@ -1126,6 +1128,12 @@ public class Query { toStringEvaluator(argEvaluators.get(0), "first argument to replace"), toStringEvaluator(argEvaluators.get(1), "second argument to replace")), "replace"); } + case REPLACE_FIRST: { + verifyArgCount(argEvaluators, 2, "replaceFirst"); + return addToken(new ReplaceFirstEvaluator(toStringEvaluator(subjectEvaluator), + toStringEvaluator(argEvaluators.get(0), "first argument to replaceFirst"), + toStringEvaluator(argEvaluators.get(1), "second argument to replaceFirst")), "replaceFirst"); + } case REPLACE_ALL: { verifyArgCount(argEvaluators, 2, "replaceAll"); return addToken(new ReplaceAllEvaluator(toStringEvaluator(subjectEvaluator), diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceFirstEvaluator.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceFirstEvaluator.java new file mode 100644 index 0000000000..934357b4b3 --- /dev/null +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/ReplaceFirstEvaluator.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language.evaluation.functions; + +import java.util.Map; +import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator; +import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult; + +public class ReplaceFirstEvaluator extends StringEvaluator { + + private final Evaluator subject; + private final Evaluator search; + private final Evaluator replacement; + + public ReplaceFirstEvaluator(final Evaluator subject, final Evaluator search, final Evaluator replacement) { + this.subject = subject; + this.search = search; + this.replacement = replacement; + } + + @Override + public QueryResult evaluate(final Map attributes) { + final String subjectValue = subject.evaluate(attributes).getValue(); + if (subjectValue == null) { + return new StringQueryResult(null); + } + final String searchValue = search.evaluate(attributes).getValue(); + final String replacementValue = replacement.evaluate(attributes).getValue(); + + return new StringQueryResult(subjectValue.replaceFirst(searchValue, replacementValue)); + } + + @Override + public Evaluator getSubjectEvaluator() { + return subject; + } + +} diff --git a/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy b/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy new file mode 100644 index 0000000000..bdd704d9b0 --- /dev/null +++ b/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.attribute.expression.language + +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult +import org.apache.nifi.expression.AttributeExpression +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@RunWith(JUnit4.class) +public class QueryGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(QueryGroovyTest.class) + + @BeforeClass + public static void setUpOnce() throws Exception { + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() { + + } + + @After + public void tearDown() { + Query.metaClass.static = null + + } + + @Test + public void testReplaceShouldReplaceAllLiteralMatches() { + // Arrange + int n = 3 + final String ORIGINAL_VALUE = "Hello World" + final Map attributes = [ + single : ORIGINAL_VALUE, + repeating: [ORIGINAL_VALUE].multiply(n).join(" ")] + logger.info("Attributes: ${attributes}") + + final String REPLACEMENT_VALUE = "Goodbye Planet" + + final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE + final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE].multiply(n).join(" ") + + final String REPLACE_LITERAL = ORIGINAL_VALUE + + final String REPLACE_SINGLE_EXPRESSION = "\${single:replace('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}" + logger.expression("Replace single | ${REPLACE_SINGLE_EXPRESSION}") + final String REPLACE_REPEATING_EXPRESSION = "\${repeating:replace('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}" + logger.expression("Replace repeating | ${REPLACE_REPEATING_EXPRESSION}") + + Query replaceSingleQuery = Query.compile(REPLACE_SINGLE_EXPRESSION) + Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION) + + // Act + QueryResult replaceSingleResult = replaceSingleQuery.evaluate(attributes) + logger.info("Replace single result: ${replaceSingleResult.value}") + + QueryResult replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes) + logger.info("Replace repeating result: ${replaceRepeatingResult.value}") + + // Assert + assert replaceSingleResult.value == EXPECTED_SINGLE_RESULT + assert replaceSingleResult.resultType == AttributeExpression.ResultType.STRING + + assert replaceRepeatingResult.value == EXPECTED_REPEATING_RESULT + assert replaceRepeatingResult.resultType == AttributeExpression.ResultType.STRING + } + + @Test + public void testReplaceFirstShouldOnlyReplaceFirstRegexMatch() { + // Arrange + int n = 3 + final String ORIGINAL_VALUE = "Hello World" + final Map attributes = [ + single : ORIGINAL_VALUE, + repeating: [ORIGINAL_VALUE].multiply(n).join(" ")] + logger.info("Attributes: ${attributes}") + + final String REPLACEMENT_VALUE = "Goodbye Planet" + + final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE + final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE, [ORIGINAL_VALUE].multiply(n - 1)].flatten().join(" ") + + final String REPLACE_ONLY_FIRST_PATTERN = /\w+\s\w+\b??/ + + final String REPLACE_SINGLE_EXPRESSION = "\${single:replaceFirst('${REPLACE_ONLY_FIRST_PATTERN}', '${REPLACEMENT_VALUE}')}" + logger.expression("Replace single | ${REPLACE_SINGLE_EXPRESSION}") + final String REPLACE_REPEATING_EXPRESSION = "\${repeating:replaceFirst('${REPLACE_ONLY_FIRST_PATTERN}', '${REPLACEMENT_VALUE}')}" + logger.expression("Replace repeating | ${REPLACE_REPEATING_EXPRESSION}") + + Query replaceSingleQuery = Query.compile(REPLACE_SINGLE_EXPRESSION) + Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION) + + // Act + QueryResult replaceSingleResult = replaceSingleQuery.evaluate(attributes) + logger.info("Replace single result: ${replaceSingleResult.value}") + + QueryResult replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes) + logger.info("Replace repeating result: ${replaceRepeatingResult.value}") + + // Assert + assert replaceSingleResult.value == EXPECTED_SINGLE_RESULT + assert replaceSingleResult.resultType == AttributeExpression.ResultType.STRING + + assert replaceRepeatingResult.value == EXPECTED_REPEATING_RESULT + assert replaceRepeatingResult.resultType == AttributeExpression.ResultType.STRING + } + + @Test + public void testReplaceFirstShouldOnlyReplaceFirstLiteralMatch() { + // Arrange + int n = 3 + final String ORIGINAL_VALUE = "Hello World" + final Map attributes = [ + single : ORIGINAL_VALUE, + repeating: [ORIGINAL_VALUE].multiply(n).join(" ")] + logger.info("Attributes: ${attributes}") + + final String REPLACEMENT_VALUE = "Goodbye Planet" + + final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE + final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE, [ORIGINAL_VALUE].multiply(n - 1)].flatten().join(" ") + + final String REPLACE_LITERAL = ORIGINAL_VALUE + + final String REPLACE_SINGLE_EXPRESSION = "\${single:replaceFirst('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}" + logger.expression("Replace single | ${REPLACE_SINGLE_EXPRESSION}") + final String REPLACE_REPEATING_EXPRESSION = "\${repeating:replaceFirst('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}" + logger.expression("Replace repeating | ${REPLACE_REPEATING_EXPRESSION}") + + Query replaceSingleQuery = Query.compile(REPLACE_SINGLE_EXPRESSION) + Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION) + + // Act + QueryResult replaceSingleResult = replaceSingleQuery.evaluate(attributes) + logger.info("Replace single result: ${replaceSingleResult.value}") + + QueryResult replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes) + logger.info("Replace repeating result: ${replaceRepeatingResult.value}") + + // Assert + assert replaceSingleResult.value == EXPECTED_SINGLE_RESULT + assert replaceSingleResult.resultType == AttributeExpression.ResultType.STRING + + assert replaceRepeatingResult.value == EXPECTED_REPEATING_RESULT + assert replaceRepeatingResult.resultType == AttributeExpression.ResultType.STRING + } + + @Test + public void testShouldDemonstrateDifferenceBetweenStringReplaceAndStringReplaceFirst() { + // Arrange + int n = 3 + final String ORIGINAL_VALUE = "Hello World" + final Map attributes = [ + single : ORIGINAL_VALUE, + repeating: [ORIGINAL_VALUE].multiply(n).join(" ")] + logger.info("Attributes: ${attributes}") + + final String REPLACEMENT_VALUE = "Goodbye Planet" + + final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE + final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE, [ORIGINAL_VALUE].multiply(n - 1)].flatten().join(" ") + + final String REPLACE_ONLY_FIRST_PATTERN = /\w+\s\w+\b??/ + + // Act + + // Execute on both single and repeating with String#replace() + String replaceSingleResult = attributes.single.replace(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE) + logger.info("Replace single result: ${replaceSingleResult}") + + String replaceRepeatingResult = attributes.repeating.replace(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE) + logger.info("Replace repeating result: ${replaceRepeatingResult}") + + // Execute on both single and repeating with String#replaceFirst() + String replaceFirstSingleResult = attributes.single.replaceFirst(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE) + logger.info("Replace first single result: ${replaceFirstSingleResult}") + + String replaceFirstRepeatingResult = attributes.repeating.replaceFirst(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE) + logger.info("Replace repeating result: ${replaceFirstRepeatingResult}") + + // Assert + assert replaceSingleResult != EXPECTED_SINGLE_RESULT + assert replaceRepeatingResult != EXPECTED_REPEATING_RESULT + + assert replaceFirstSingleResult == EXPECTED_SINGLE_RESULT + assert replaceFirstRepeatingResult == EXPECTED_REPEATING_RESULT + } +} \ No newline at end of file diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc index 1580037905..9c08ef445e 100644 --- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc +++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc @@ -831,7 +831,7 @@ then the following Expressions will result in the following values: [.function] === replace -*Description*: [.description]#Replaces occurrences of one String within the Subject with another String.# +*Description*: [.description]#Replaces *all* occurrences of one literal String within the Subject with another String.# *Subject Type*: [.subject]#String# @@ -860,10 +860,42 @@ Expressions will provide the following results: +[.function] +=== replaceFirst + +*Description*: [.description]#Replaces *the first* occurrence of one literal String or regular expression within the Subject with another String.# + +*Subject Type*: [.subject]#String# + +*Arguments*: + + - [.argName]#_Search String_# : [.argDesc]#The String (literal or regular expression pattern) to find within the Subject# + - [.argName]#_Replacement_# : [.argDesc]#The value to replace _Search String_ with# + +*Return Type*: [.returnType]#String# + +*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the following +Expressions will provide the following results: + + + +.ReplaceFirst Examples +|=================================================================== +| Expression | Value +| `${filename:replaceFirst('a', 'the')}` | `the brand new filename.txt` +| `${filename:replaceFirst('[br]', 'g')}` | `a grand new filename.txt` +| `${filename:replaceFirst('XYZ', 'ZZZ')}` | `a brand new filename.txt` +| `${filename:replaceFirst('\w{8}', 'book')}` | `a brand new book.txt` +|=================================================================== + + + + + [.function] === replaceAll -*Description*: [.description]#The `replaceAll` function takes two String arguments: a Regular Expression (NiFi uses the Java Pattern +*Description*: [.description]#The `replaceAll` function takes two String arguments: a literal String or Regular Expression (NiFi uses the Java Pattern syntax), and a replacement string. The return value is the result of substituting the replacement string for all patterns within the Subject that match the Regular Expression.# @@ -884,7 +916,7 @@ Expressions will provide the following results: -.replaceAll Examples +.ReplaceAll Examples |======================================================================================= | Expression | Value | `${filename:replaceAll('\..*', '')}` | `a brand new filename`