From 52b24b93d9f7763744c792c0cfed8974f8e6cb83 Mon Sep 17 00:00:00 2001 From: Joe Skora Date: Wed, 18 Nov 2015 19:09:27 -0500 Subject: [PATCH] NIFI-1123 Adds expression language support to DeleteAttributesExpression on UpdateAttributes Processor. Reviewed by Tony Kurc (trkurc@gmail.com) after Aldrin Piri did the initial review and actionable comments --- .../attributes/UpdateAttribute.java | 35 ++++- .../attributes/TestUpdateAttribute.java | 130 ++++++++++++++++++ 2 files changed, 163 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java index 8cf5726c49..9209f6dbab 100644 --- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java +++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/main/java/org/apache/nifi/processors/attributes/UpdateAttribute.java @@ -45,6 +45,7 @@ import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; import org.apache.nifi.expression.AttributeExpression; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; @@ -131,12 +132,40 @@ public class UpdateAttribute extends AbstractProcessor implements Searchable { private final Set relationships; + private static final Validator DELETE_PROPERTY_VALIDATOR = new Validator() { + private static final Validator DPV_RE_VALIDATOR = StandardValidators.createRegexValidator(0, Integer.MAX_VALUE, true); + @Override + public ValidationResult validate(String subject, String input, ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + final AttributeExpression.ResultType resultType = context.newExpressionLanguageCompiler().getResultType(input); + if (!resultType.equals(AttributeExpression.ResultType.STRING)) { + return new ValidationResult.Builder() + .subject(subject) + .input(input) + .valid(false) + .explanation("Expected property to to return type " + AttributeExpression.ResultType.STRING + + " but expression returns type " + resultType) + .build(); + } + return new ValidationResult.Builder() + .subject(subject) + .input(input) + .valid(true) + .explanation("Property returns type " + AttributeExpression.ResultType.STRING) + .build(); + } + + return DPV_RE_VALIDATOR.validate(subject, input, context); + } + }; + // static properties public static final PropertyDescriptor DELETE_ATTRIBUTES = new PropertyDescriptor.Builder() .name("Delete Attributes Expression") .description("Regular expression for attributes to be deleted from flowfiles.") .required(false) - .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .addValidator(DELETE_PROPERTY_VALIDATOR) + .expressionLanguageSupported(true) .build(); // relationships @@ -477,7 +506,9 @@ public class UpdateAttribute extends AbstractProcessor implements Searchable { } } else { try { - final String regex = action.getValue(); + final String actionValue = action.getValue(); + final String regex = (actionValue == null) ? null : + getPropertyValue(actionValue, context).evaluateAttributeExpressions(flowfile).getValue(); if (regex != null) { Pattern pattern = Pattern.compile(regex); final Set attributeKeys = flowfile.getAttributes().keySet(); diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java index f1b75edf21..90b51bd6b8 100644 --- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java +++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-processor/src/test/java/org/apache/nifi/update/attributes/TestUpdateAttribute.java @@ -24,6 +24,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.regex.PatternSyntaxException; import org.apache.nifi.processors.attributes.UpdateAttribute; import org.apache.nifi.update.attributes.serde.CriteriaSerDe; @@ -33,6 +34,8 @@ import org.apache.nifi.util.TestRunners; import org.junit.Test; +import static org.junit.Assert.assertEquals; + /** * */ @@ -500,4 +503,131 @@ public class TestUpdateAttribute { result.get(0).assertAttributeNotExists("sample.1"); result.get(0).assertAttributeExists("simple.1"); } + + @Test + public void testAttributeKey() { + final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute()); + runner.setProperty("Delete Attributes Expression", "(attribute\\.[2-5]|sample.*)"); + + final Map attributes = new HashMap<>(); + attributes.put("attribute.1", "value.1"); + attributes.put("attribute.2", "value.2"); + attributes.put("attribute.6", "value.6"); + attributes.put("sampleSize", "value.size"); + attributes.put("sample.1", "value.sample.1"); + attributes.put("simple.1", "value.simple.1"); + runner.enqueue(new byte[0], attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1); + final List result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS); + result.get(0).assertAttributeEquals("attribute.1", "value.1"); + result.get(0).assertAttributeNotExists("attribute.2"); + result.get(0).assertAttributeExists("attribute.6"); + result.get(0).assertAttributeNotExists("sampleSize"); + result.get(0).assertAttributeNotExists("sample.1"); + result.get(0).assertAttributeExists("simple.1"); + } + + @Test + public void testExpressionLiteralDelete() { + final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute()); + runner.setProperty("Delete Attributes Expression", "${literal('attribute\\.'):append(${literal(6)})}"); + + final Map attributes = new HashMap<>(); + attributes.put("attribute.1", "value.1"); + attributes.put("attribute.2", "value.2"); + attributes.put("attribute.6", "value.6"); + attributes.put("sampleSize", "value.size"); + attributes.put("sample.1", "value.sample.1"); + attributes.put("simple.1", "value.simple.1"); + runner.enqueue(new byte[0], attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1); + final List result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS); + result.get(0).assertAttributeEquals("attribute.1", "value.1"); + result.get(0).assertAttributeExists("attribute.2"); + result.get(0).assertAttributeNotExists("attribute.6"); + result.get(0).assertAttributeExists("sampleSize"); + result.get(0).assertAttributeExists("sample.1"); + result.get(0).assertAttributeExists("simple.1"); + } + + @Test + public void testExpressionRegexDelete() { + final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute()); + runner.setProperty("Delete Attributes Expression", "${literal('(attribute\\.'):append(${literal('[2-5]')}):append(${literal('|sample.*)')})}"); + + final Map attributes = new HashMap<>(); + attributes.put("attribute.1", "value.1"); + attributes.put("attribute.2", "value.2"); + attributes.put("attribute.6", "value.6"); + attributes.put("sampleSize", "value.size"); + attributes.put("sample.1", "value.sample.1"); + attributes.put("simple.1", "value.simple.1"); + runner.enqueue(new byte[0], attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1); + final List result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS); + result.get(0).assertAttributeEquals("attribute.1", "value.1"); + result.get(0).assertAttributeNotExists("attribute.2"); + result.get(0).assertAttributeExists("attribute.6"); + result.get(0).assertAttributeNotExists("sampleSize"); + result.get(0).assertAttributeNotExists("sample.1"); + result.get(0).assertAttributeExists("simple.1"); + } + + @Test + public void testAttributeListDelete() { + final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute()); + runner.setProperty("Delete Attributes Expression", "attribute.1|attribute.2|sample.1|simple.1"); + + final Map attributes = new HashMap<>(); + attributes.put("attribute.1", "value.1"); + attributes.put("attribute.2", "value.2"); + attributes.put("attribute.6", "value.6"); + attributes.put("sampleSize", "value.size"); + attributes.put("sample.1", "value.sample.1"); + attributes.put("simple.1", "value.simple.1"); + runner.enqueue(new byte[0], attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(UpdateAttribute.REL_SUCCESS, 1); + final List result = runner.getFlowFilesForRelationship(UpdateAttribute.REL_SUCCESS); + result.get(0).assertAttributeNotExists("attribute.1"); + result.get(0).assertAttributeNotExists("attribute.2"); + result.get(0).assertAttributeExists("attribute.6"); + result.get(0).assertAttributeExists("sampleSize"); + result.get(0).assertAttributeNotExists("sample.1"); + result.get(0).assertAttributeNotExists("simple.1"); + } + + @Test + public void testInvalidRegex() { + final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute()); + runner.setProperty("Delete Attributes Expression", "("); + runner.assertNotValid(); + } + + @Test + public void testInvalidRegexInAttribute() { + final TestRunner runner = TestRunners.newTestRunner(new UpdateAttribute()); + runner.setProperty("Delete Attributes Expression", "${butter}"); + runner.assertValid(); + + final Map attributes = new HashMap<>(); + attributes.put("butter", "("); + runner.enqueue(new byte[0], attributes); + try { + runner.run(); + } catch (Throwable t) { + assertEquals(t.getCause().getClass(), PatternSyntaxException.class); + } + } }