From 8a197e5a96479b4b298aa821a8ab9fdfdc1deb60 Mon Sep 17 00:00:00 2001 From: Patrice Freydiere Date: Sat, 9 Sep 2017 16:43:14 +0200 Subject: [PATCH] NIFI-4367 Fix on processor for permit deriving script classes from AbstractProcessor or other Records based base classes Improve UT Add licence and fix import improve filtering properties Signed-off-by: Matthew Burgess This closes #2201 --- .../script/InvokeScriptedProcessor.java | 41 ++++--- ...redPropertiesValidationContextAdapter.java | 72 ++++++++++++ .../script/impl/ValidationContextAdapter.java | 97 ++++++++++++++++ .../processors/script/TestInvokeGroovy.java | 30 +++++ .../test_implementingabstractProcessor.groovy | 108 ++++++++++++++++++ 5 files changed, 331 insertions(+), 17 deletions(-) create mode 100644 nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/FilteredPropertiesValidationContextAdapter.java create mode 100644 nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ValidationContextAdapter.java create mode 100644 nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_implementingabstractProcessor.groovy diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java index 45439bf36f..c2df1282d2 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java @@ -16,6 +16,22 @@ */ package org.apache.nifi.processors.script; +import java.io.File; +import java.io.FileInputStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import javax.script.Invocable; +import javax.script.ScriptEngine; +import javax.script.ScriptException; + import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.annotation.behavior.DynamicProperty; @@ -46,21 +62,7 @@ import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.script.ScriptingComponentHelper; import org.apache.nifi.script.ScriptingComponentUtils; - -import javax.script.Invocable; -import javax.script.ScriptEngine; -import javax.script.ScriptException; -import java.io.File; -import java.io.FileInputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; +import org.apache.nifi.script.impl.FilteredPropertiesValidationContextAdapter; @Tags({"script", "invoke", "groovy", "python", "jython", "jruby", "ruby", "javascript", "js", "lua", "luaj"}) @CapabilityDescription("Experimental - Invokes a script engine for a Processor defined in the given script. The script must define " @@ -477,8 +479,13 @@ public class InvokeScriptedProcessor extends AbstractSessionFactoryProcessor { // if there was existing validation errors and the processor loaded successfully if (currentValidationResults.isEmpty() && instance != null) { try { - // defer to the underlying processor for validation - final Collection instanceResults = instance.validate(context); + // defer to the underlying processor for validation, without the + // invokescriptedprocessor properties + final Set innerPropertyDescriptor = new HashSet(scriptingComponentHelper.getDescriptors()); + + ValidationContext innerValidationContext = new FilteredPropertiesValidationContextAdapter(context, innerPropertyDescriptor); + final Collection instanceResults = instance.validate(innerValidationContext); + if (instanceResults != null && instanceResults.size() > 0) { // return the validation results from the underlying instance return instanceResults; diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/FilteredPropertiesValidationContextAdapter.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/FilteredPropertiesValidationContextAdapter.java new file mode 100644 index 0000000000..440a2a7139 --- /dev/null +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/FilteredPropertiesValidationContextAdapter.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.script.impl; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.components.ValidationContext; + +/** + * Filters properties in the ValidationContext, proxy approach, for removing unwanted properties + * + */ +public class FilteredPropertiesValidationContextAdapter extends ValidationContextAdapter { + + private HashMap properties; + + public FilteredPropertiesValidationContextAdapter(ValidationContext validationContext, Set removedProperties) { + super(validationContext); + properties = new HashMap<>(); + Map parentProperties = super.getProperties(); + if (parentProperties != null) { + for (Map.Entry propertyEntry: parentProperties.entrySet()) { + if (!removedProperties.contains(propertyEntry.getKey())) { + properties.put(propertyEntry.getKey(), propertyEntry.getValue()); + } + } + } + } + + @Override + public Map getAllProperties() { + final Map propValueMap = new LinkedHashMap<>(); + for (final Map.Entry entry : getProperties().entrySet()) { + propValueMap.put(entry.getKey().getName(), entry.getValue()); + } + return propValueMap; + } + + @Override + public Map getProperties() { + return Collections.unmodifiableMap(properties); + } + + @Override + public PropertyValue getProperty(PropertyDescriptor descriptor) { + if (properties.keySet().contains(descriptor)) { + return super.getProperty(descriptor); + } + return null; + } + +} diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ValidationContextAdapter.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ValidationContextAdapter.java new file mode 100644 index 0000000000..20b3e98e7f --- /dev/null +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ValidationContextAdapter.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.script.impl; + +import java.util.Map; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.controller.ControllerServiceLookup; +import org.apache.nifi.expression.ExpressionLanguageCompiler; + +public abstract class ValidationContextAdapter implements ValidationContext { + + private ValidationContext innerValidationContext; + + public ValidationContextAdapter(ValidationContext innerValidationContext) { + assert innerValidationContext != null; + this.innerValidationContext = innerValidationContext; + } + + @Override + public PropertyValue getProperty(PropertyDescriptor descriptor) { + return innerValidationContext.getProperty(descriptor); + } + + @Override + public Map getAllProperties() { + return innerValidationContext.getAllProperties(); + } + + @Override + public ControllerServiceLookup getControllerServiceLookup() { + return innerValidationContext.getControllerServiceLookup(); + } + + @Override + public ValidationContext getControllerServiceValidationContext(ControllerService controllerService) { + return innerValidationContext.getControllerServiceValidationContext(controllerService); + } + + @Override + public ExpressionLanguageCompiler newExpressionLanguageCompiler() { + return innerValidationContext.newExpressionLanguageCompiler(); + } + + @Override + public PropertyValue newPropertyValue(String value) { + return innerValidationContext.newPropertyValue(value); + } + + @Override + public Map getProperties() { + return innerValidationContext.getProperties(); + } + + @Override + public String getAnnotationData() { + return innerValidationContext.getAnnotationData(); + } + + @Override + public boolean isValidationRequired(ControllerService service) { + return innerValidationContext.isValidationRequired(service); + } + + @Override + public boolean isExpressionLanguagePresent(String value) { + return innerValidationContext.isExpressionLanguagePresent(value); + } + + @Override + public boolean isExpressionLanguageSupported(String propertyName) { + return innerValidationContext.isExpressionLanguageSupported(propertyName); + } + + @Override + public String getProcessGroupIdentifier() { + return innerValidationContext.getProcessGroupIdentifier(); + } + +} diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java index 4a17c8ade6..64047ae747 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.processors.script; +import org.apache.commons.codec.binary.Hex; + import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.Relationship; import org.apache.nifi.script.ScriptingComponentUtils; @@ -29,6 +31,7 @@ import org.junit.Before; import org.junit.Test; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; import java.util.List; import java.util.Set; @@ -36,6 +39,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; + public class TestInvokeGroovy extends BaseScriptTest { @Before @@ -179,4 +183,30 @@ public class TestInvokeGroovy extends BaseScriptTest { runner.setProperty("test-attribute", "test"); runner.assertValid(); } + + /** + * Tests a script that derive from AbstractProcessor as base class + * + * @throws Exception Any error encountered while testing + */ + @Test + public void testAbstractProcessorImplementationWithBodyScriptFile() throws Exception { + runner.setValidateExpressionUsage(false); + runner.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE, "Groovy"); + runner.setProperty(ScriptingComponentUtils.SCRIPT_BODY, getFileContentsAsString(TEST_RESOURCE_LOCATION + "groovy/test_implementingabstractProcessor.groovy")); + runner.setProperty(ScriptingComponentUtils.MODULES, TEST_RESOURCE_LOCATION + "groovy"); + runner.setProperty("custom_prop", "bla bla"); + + runner.assertValid(); + runner.enqueue("test".getBytes(StandardCharsets.UTF_8)); + runner.run(); + + runner.assertAllFlowFilesTransferred("success", 1); + final List result = runner.getFlowFilesForRelationship("success"); + assertTrue(result.size() == 1); + final String expectedOutput = new String(Hex.encodeHex(MessageDigest.getInstance("MD5").digest("testbla bla".getBytes()))); + final MockFlowFile outputFlowFile = result.get(0); + outputFlowFile.assertContentEquals(expectedOutput); + outputFlowFile.assertAttributeEquals("outAttr", expectedOutput); + } } diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_implementingabstractProcessor.groovy b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_implementingabstractProcessor.groovy new file mode 100644 index 0000000000..1c00879e15 --- /dev/null +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_implementingabstractProcessor.groovy @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.apache.commons.codec.binary.Hex + +import org.apache.nifi.flowfile.FlowFile +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult +import org.apache.nifi.processor.ProcessContext +import org.apache.nifi.processor.ProcessSession +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException +import org.apache.nifi.processor.io.InputStreamCallback +import org.apache.nifi.processor.io.OutputStreamCallback +import org.apache.nifi.processor.util.StandardValidators +import org.apache.nifi.stream.io.StreamUtils + +import java.security.MessageDigest + + +class TestAbstractProcessor extends AbstractProcessor { + + def REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("FlowFiles that were successfully processed") + .build(); + + final static String attr = "outAttr"; + + final static PropertyDescriptor myCustomProp = new PropertyDescriptor.Builder() + .name("custom_prop") + .displayName("a custom prop") + .description("bla bla bla") + .expressionLanguageSupported(false) + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + @Override + List getSupportedPropertyDescriptors(){ + return [myCustomProp] as List + } + + @Override + Set getRelationships() { + return [REL_SUCCESS] as Set + } + + @Override + Collection customValidate(ValidationContext context) { + final String myCustomPropValue = context.getProperty(myCustomProp).getValue() + if (myCustomPropValue.length() < 3) { + final List problems = new ArrayList<>(1); + problems.add(new ValidationResult.Builder().subject(myCustomProp.getName()) + .input(myCustomPropValue) + .valid(false) + .explanation("myCustomProp should contain a string with at least 3 chars.") + .build()); + return problems; + } + return super.customValidate(context) + } + + @Override + void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + FlowFile requestFlowFile = session.get(); + String myCustomPropValue = context.getProperty(myCustomProp).evaluateAttributeExpressions(requestFlowFile).getValue(); + final byte[] buff = new byte[(int) requestFlowFile.getSize()]; + session.read(requestFlowFile, new InputStreamCallback() { + @Override + void process(InputStream inp) throws IOException { + StreamUtils.fillBuffer(inp, buff); + } + }); + final String content = new String(buff); + final String md5 = generateMD5_A(content + myCustomPropValue); + requestFlowFile = session.putAttribute(requestFlowFile, attr, md5); + requestFlowFile = session.write(requestFlowFile, new OutputStreamCallback() { + @Override + public void process(OutputStream out) throws IOException { + out.write(md5.getBytes()); + } + }); + + session.transfer(requestFlowFile, REL_SUCCESS); + } + + static def generateMD5_A(String s){ + new String(Hex.encodeHex(MessageDigest.getInstance("MD5").digest(s.bytes))); + } +} + +processor = new TestAbstractProcessor();