NIFI-1919 Added replaceFirst() expression language method which accepts literal or pattern for replacement.

Reverted whitespace changes. (+8 squashed commits)
Squashed commits:
[329755c] NIFI-1919 Reverted import re-organization from IDE.
[cf73c2f] NIFI-1919 Updated expression language guide.
[d9a1455] NIFI-1919 Reverted changes to ReplaceEvaluator.
Added ReplaceFirstEvaluator.
Added replace first logic to Query buildFunctionEvaluator.
Added unit tests.
[e2eb880] NIFI-1919 Added replaceFirst to AttributeExpression lexer and parser grammar definitions.
[11fe913] NIFI-1919 Ignored demonstrative test for replaceAll as it behaves as expected.
[af97be1] NIFI-1919 Changed ReplaceEvaluator to use String#replaceFirst which interprets regex instead of compiling as literal.
Demonstrative unit test now passes but two existing unit tests fail. I am not sure these tests are correct.
[f24f17b] NIFI-1919 Added working unit test to illustrate fix.
[8a0d43b] NIFI-1919 Added Groovy unit test to demonstrate issue.
Added DelegatingMetaClass code to record it (test not complete).

This closes .

Signed-off-by: Andy LoPresto <alopresto@apache.org>
This commit is contained in:
Andy LoPresto 2016-05-24 19:05:34 -07:00
parent bcead3500d
commit 8127314975
No known key found for this signature in database
GPG Key ID: 3C6EF65B2F7DEF69
6 changed files with 313 additions and 6 deletions
nifi-commons/nifi-expression-language/src
main
antlr3/org/apache/nifi/attribute/expression/language/antlr
java/org/apache/nifi/attribute/expression/language
test/groovy/org/apache/nifi/attribute/expression/language
nifi-docs/src/main/asciidoc

View File

@ -156,6 +156,7 @@ TO_LITERAL : 'literal';
// 2 arg functions
SUBSTRING : 'substring';
REPLACE : 'replace';
REPLACE_FIRST : 'replaceFirst';
REPLACE_ALL : 'replaceAll';
// 4 arg functions

View File

@ -77,7 +77,7 @@ zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECOD
oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN) LPAREN! anyArg RPAREN!) |
(TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
twoArgString : ((REPLACE | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) |
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) |
(SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
fiveArgString : GET_DELIMITED_FIELD LPAREN! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg)?)?)?)? RPAREN!;

View File

@ -79,6 +79,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.Random
import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceAllEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEmptyEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceFirstEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.ReplaceNullEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.StartsWithEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.StringToDateEvaluator;
@ -172,6 +173,7 @@ import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpre
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.RANDOM;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_ALL;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_EMPTY;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_FIRST;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_NULL;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STARTS_WITH;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STRING_LITERAL;
@ -427,8 +429,8 @@ public class Query {
}
static Map<String, String> createExpressionMap(final FlowFile flowFile, final Map<String, String> additionalAttributes) {
final Map<String, String> attributeMap = flowFile == null ? Collections.<String, String> emptyMap() : flowFile.getAttributes();
final Map<String, String> additionalOrEmpty = additionalAttributes == null ? Collections.<String, String> emptyMap() : additionalAttributes;
final Map<String, String> attributeMap = flowFile == null ? Collections.emptyMap() : flowFile.getAttributes();
final Map<String, String> additionalOrEmpty = additionalAttributes == null ? Collections.emptyMap() : additionalAttributes;
final Map<String, String> envMap = System.getenv();
final Map<?, ?> sysProps = System.getProperties();
@ -1126,6 +1128,12 @@ public class Query {
toStringEvaluator(argEvaluators.get(0), "first argument to replace"),
toStringEvaluator(argEvaluators.get(1), "second argument to replace")), "replace");
}
case REPLACE_FIRST: {
verifyArgCount(argEvaluators, 2, "replaceFirst");
return addToken(new ReplaceFirstEvaluator(toStringEvaluator(subjectEvaluator),
toStringEvaluator(argEvaluators.get(0), "first argument to replaceFirst"),
toStringEvaluator(argEvaluators.get(1), "second argument to replaceFirst")), "replaceFirst");
}
case REPLACE_ALL: {
verifyArgCount(argEvaluators, 2, "replaceAll");
return addToken(new ReplaceAllEvaluator(toStringEvaluator(subjectEvaluator),

View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.attribute.expression.language.evaluation.functions;
import java.util.Map;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
public class ReplaceFirstEvaluator extends StringEvaluator {
private final Evaluator<String> subject;
private final Evaluator<String> search;
private final Evaluator<String> replacement;
public ReplaceFirstEvaluator(final Evaluator<String> subject, final Evaluator<String> search, final Evaluator<String> replacement) {
this.subject = subject;
this.search = search;
this.replacement = replacement;
}
@Override
public QueryResult<String> evaluate(final Map<String, String> attributes) {
final String subjectValue = subject.evaluate(attributes).getValue();
if (subjectValue == null) {
return new StringQueryResult(null);
}
final String searchValue = search.evaluate(attributes).getValue();
final String replacementValue = replacement.evaluate(attributes).getValue();
return new StringQueryResult(subjectValue.replaceFirst(searchValue, replacementValue));
}
@Override
public Evaluator<?> getSubjectEvaluator() {
return subject;
}
}

View File

@ -0,0 +1,212 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.attribute.expression.language
import org.apache.nifi.attribute.expression.language.evaluation.QueryResult
import org.apache.nifi.expression.AttributeExpression
import org.junit.After
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
@RunWith(JUnit4.class)
public class QueryGroovyTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(QueryGroovyTest.class)
@BeforeClass
public static void setUpOnce() throws Exception {
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
public void setUp() {
}
@After
public void tearDown() {
Query.metaClass.static = null
}
@Test
public void testReplaceShouldReplaceAllLiteralMatches() {
// Arrange
int n = 3
final String ORIGINAL_VALUE = "Hello World"
final Map<String, String> attributes = [
single : ORIGINAL_VALUE,
repeating: [ORIGINAL_VALUE].multiply(n).join(" ")]
logger.info("Attributes: ${attributes}")
final String REPLACEMENT_VALUE = "Goodbye Planet"
final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE
final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE].multiply(n).join(" ")
final String REPLACE_LITERAL = ORIGINAL_VALUE
final String REPLACE_SINGLE_EXPRESSION = "\${single:replace('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}"
logger.expression("Replace single | ${REPLACE_SINGLE_EXPRESSION}")
final String REPLACE_REPEATING_EXPRESSION = "\${repeating:replace('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}"
logger.expression("Replace repeating | ${REPLACE_REPEATING_EXPRESSION}")
Query replaceSingleQuery = Query.compile(REPLACE_SINGLE_EXPRESSION)
Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION)
// Act
QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(attributes)
logger.info("Replace single result: ${replaceSingleResult.value}")
QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes)
logger.info("Replace repeating result: ${replaceRepeatingResult.value}")
// Assert
assert replaceSingleResult.value == EXPECTED_SINGLE_RESULT
assert replaceSingleResult.resultType == AttributeExpression.ResultType.STRING
assert replaceRepeatingResult.value == EXPECTED_REPEATING_RESULT
assert replaceRepeatingResult.resultType == AttributeExpression.ResultType.STRING
}
@Test
public void testReplaceFirstShouldOnlyReplaceFirstRegexMatch() {
// Arrange
int n = 3
final String ORIGINAL_VALUE = "Hello World"
final Map<String, String> attributes = [
single : ORIGINAL_VALUE,
repeating: [ORIGINAL_VALUE].multiply(n).join(" ")]
logger.info("Attributes: ${attributes}")
final String REPLACEMENT_VALUE = "Goodbye Planet"
final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE
final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE, [ORIGINAL_VALUE].multiply(n - 1)].flatten().join(" ")
final String REPLACE_ONLY_FIRST_PATTERN = /\w+\s\w+\b??/
final String REPLACE_SINGLE_EXPRESSION = "\${single:replaceFirst('${REPLACE_ONLY_FIRST_PATTERN}', '${REPLACEMENT_VALUE}')}"
logger.expression("Replace single | ${REPLACE_SINGLE_EXPRESSION}")
final String REPLACE_REPEATING_EXPRESSION = "\${repeating:replaceFirst('${REPLACE_ONLY_FIRST_PATTERN}', '${REPLACEMENT_VALUE}')}"
logger.expression("Replace repeating | ${REPLACE_REPEATING_EXPRESSION}")
Query replaceSingleQuery = Query.compile(REPLACE_SINGLE_EXPRESSION)
Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION)
// Act
QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(attributes)
logger.info("Replace single result: ${replaceSingleResult.value}")
QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes)
logger.info("Replace repeating result: ${replaceRepeatingResult.value}")
// Assert
assert replaceSingleResult.value == EXPECTED_SINGLE_RESULT
assert replaceSingleResult.resultType == AttributeExpression.ResultType.STRING
assert replaceRepeatingResult.value == EXPECTED_REPEATING_RESULT
assert replaceRepeatingResult.resultType == AttributeExpression.ResultType.STRING
}
@Test
public void testReplaceFirstShouldOnlyReplaceFirstLiteralMatch() {
// Arrange
int n = 3
final String ORIGINAL_VALUE = "Hello World"
final Map<String, String> attributes = [
single : ORIGINAL_VALUE,
repeating: [ORIGINAL_VALUE].multiply(n).join(" ")]
logger.info("Attributes: ${attributes}")
final String REPLACEMENT_VALUE = "Goodbye Planet"
final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE
final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE, [ORIGINAL_VALUE].multiply(n - 1)].flatten().join(" ")
final String REPLACE_LITERAL = ORIGINAL_VALUE
final String REPLACE_SINGLE_EXPRESSION = "\${single:replaceFirst('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}"
logger.expression("Replace single | ${REPLACE_SINGLE_EXPRESSION}")
final String REPLACE_REPEATING_EXPRESSION = "\${repeating:replaceFirst('${REPLACE_LITERAL}', '${REPLACEMENT_VALUE}')}"
logger.expression("Replace repeating | ${REPLACE_REPEATING_EXPRESSION}")
Query replaceSingleQuery = Query.compile(REPLACE_SINGLE_EXPRESSION)
Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION)
// Act
QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(attributes)
logger.info("Replace single result: ${replaceSingleResult.value}")
QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes)
logger.info("Replace repeating result: ${replaceRepeatingResult.value}")
// Assert
assert replaceSingleResult.value == EXPECTED_SINGLE_RESULT
assert replaceSingleResult.resultType == AttributeExpression.ResultType.STRING
assert replaceRepeatingResult.value == EXPECTED_REPEATING_RESULT
assert replaceRepeatingResult.resultType == AttributeExpression.ResultType.STRING
}
@Test
public void testShouldDemonstrateDifferenceBetweenStringReplaceAndStringReplaceFirst() {
// Arrange
int n = 3
final String ORIGINAL_VALUE = "Hello World"
final Map<String, String> attributes = [
single : ORIGINAL_VALUE,
repeating: [ORIGINAL_VALUE].multiply(n).join(" ")]
logger.info("Attributes: ${attributes}")
final String REPLACEMENT_VALUE = "Goodbye Planet"
final String EXPECTED_SINGLE_RESULT = REPLACEMENT_VALUE
final String EXPECTED_REPEATING_RESULT = [REPLACEMENT_VALUE, [ORIGINAL_VALUE].multiply(n - 1)].flatten().join(" ")
final String REPLACE_ONLY_FIRST_PATTERN = /\w+\s\w+\b??/
// Act
// Execute on both single and repeating with String#replace()
String replaceSingleResult = attributes.single.replace(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE)
logger.info("Replace single result: ${replaceSingleResult}")
String replaceRepeatingResult = attributes.repeating.replace(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE)
logger.info("Replace repeating result: ${replaceRepeatingResult}")
// Execute on both single and repeating with String#replaceFirst()
String replaceFirstSingleResult = attributes.single.replaceFirst(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE)
logger.info("Replace first single result: ${replaceFirstSingleResult}")
String replaceFirstRepeatingResult = attributes.repeating.replaceFirst(REPLACE_ONLY_FIRST_PATTERN, REPLACEMENT_VALUE)
logger.info("Replace repeating result: ${replaceFirstRepeatingResult}")
// Assert
assert replaceSingleResult != EXPECTED_SINGLE_RESULT
assert replaceRepeatingResult != EXPECTED_REPEATING_RESULT
assert replaceFirstSingleResult == EXPECTED_SINGLE_RESULT
assert replaceFirstRepeatingResult == EXPECTED_REPEATING_RESULT
}
}

View File

@ -831,7 +831,7 @@ then the following Expressions will result in the following values:
[.function]
=== replace
*Description*: [.description]#Replaces occurrences of one String within the Subject with another String.#
*Description*: [.description]#Replaces *all* occurrences of one literal String within the Subject with another String.#
*Subject Type*: [.subject]#String#
@ -860,10 +860,42 @@ Expressions will provide the following results:
[.function]
=== replaceFirst
*Description*: [.description]#Replaces *the first* occurrence of one literal String or regular expression within the Subject with another String.#
*Subject Type*: [.subject]#String#
*Arguments*:
- [.argName]#_Search String_# : [.argDesc]#The String (literal or regular expression pattern) to find within the Subject#
- [.argName]#_Replacement_# : [.argDesc]#The value to replace _Search String_ with#
*Return Type*: [.returnType]#String#
*Examples*: If the "filename" attribute has the value "a brand new filename.txt", then the following
Expressions will provide the following results:
.ReplaceFirst Examples
|===================================================================
| Expression | Value
| `${filename:replaceFirst('a', 'the')}` | `the brand new filename.txt`
| `${filename:replaceFirst('[br]', 'g')}` | `a grand new filename.txt`
| `${filename:replaceFirst('XYZ', 'ZZZ')}` | `a brand new filename.txt`
| `${filename:replaceFirst('\w{8}', 'book')}` | `a brand new book.txt`
|===================================================================
[.function]
=== replaceAll
*Description*: [.description]#The `replaceAll` function takes two String arguments: a Regular Expression (NiFi uses the Java Pattern
*Description*: [.description]#The `replaceAll` function takes two String arguments: a literal String or Regular Expression (NiFi uses the Java Pattern
syntax), and a replacement string. The return value is the result of substituting the replacement string for
all patterns within the Subject that match the Regular Expression.#
@ -884,7 +916,7 @@ Expressions will provide the following results:
.replaceAll Examples
.ReplaceAll Examples
|=======================================================================================
| Expression | Value
| `${filename:replaceAll('\..*', '')}` | `a brand new filename`