NIFI-4367 Fix on processor for permit deriving script classes from AbstractProcessor or other Records based base classes

Improve UT

Add licence and fix import

improve filtering properties

Signed-off-by: Matthew Burgess <mattyb149@apache.org>

This closes #2201

nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java

@@ -16,6 +16,22 @@
  */
 package org.apache.nifi.processors.script;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.script.Invocable;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.annotation.behavior.DynamicProperty;
@@ -46,21 +62,7 @@ import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.util.StandardValidators;
 import org.apache.nifi.script.ScriptingComponentHelper;
 import org.apache.nifi.script.ScriptingComponentUtils;
-
-import javax.script.Invocable;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
+import org.apache.nifi.script.impl.FilteredPropertiesValidationContextAdapter;
 
 @Tags({"script", "invoke", "groovy", "python", "jython", "jruby", "ruby", "javascript", "js", "lua", "luaj"})
 @CapabilityDescription("Experimental - Invokes a script engine for a Processor defined in the given script. The script must define "
@@ -477,8 +479,13 @@ public class InvokeScriptedProcessor extends AbstractSessionFactoryProcessor {
         // if there was existing validation errors and the processor loaded successfully
         if (currentValidationResults.isEmpty() && instance != null) {
             try {
-                // defer to the underlying processor for validation
-                final Collection<ValidationResult> instanceResults = instance.validate(context);
+                // defer to the underlying processor for validation, without the
+                // invokescriptedprocessor properties
+                final Set<PropertyDescriptor> innerPropertyDescriptor = new HashSet<PropertyDescriptor>(scriptingComponentHelper.getDescriptors());
+
+                ValidationContext innerValidationContext = new FilteredPropertiesValidationContextAdapter(context, innerPropertyDescriptor);
+                final Collection<ValidationResult> instanceResults = instance.validate(innerValidationContext);
+
                 if (instanceResults != null && instanceResults.size() > 0) {
                     // return the validation results from the underlying instance
                     return instanceResults;

nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/FilteredPropertiesValidationContextAdapter.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.script.impl;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.ValidationContext;
+
+/**
+ * Filters properties in the ValidationContext, proxy approach, for removing unwanted properties
+ *
+ */
+public class FilteredPropertiesValidationContextAdapter extends ValidationContextAdapter {
+
+    private HashMap<PropertyDescriptor, String> properties;
+
+    public FilteredPropertiesValidationContextAdapter(ValidationContext validationContext, Set<PropertyDescriptor> removedProperties) {
+        super(validationContext);
+        properties = new HashMap<>();
+        Map<PropertyDescriptor, String> parentProperties = super.getProperties();
+        if (parentProperties != null) {
+            for (Map.Entry<PropertyDescriptor, String> propertyEntry: parentProperties.entrySet()) {
+                if (!removedProperties.contains(propertyEntry.getKey())) {
+                    properties.put(propertyEntry.getKey(), propertyEntry.getValue());
+                }
+            }
+        }
+    }
+
+    @Override
+    public Map<String, String> getAllProperties() {
+        final Map<String,String> propValueMap = new LinkedHashMap<>();
+        for (final Map.Entry<PropertyDescriptor, String> entry : getProperties().entrySet()) {
+            propValueMap.put(entry.getKey().getName(), entry.getValue());
+        }
+        return propValueMap;
+    }
+
+    @Override
+    public Map<PropertyDescriptor, String> getProperties() {
+        return Collections.unmodifiableMap(properties);
+    }
+
+    @Override
+    public PropertyValue getProperty(PropertyDescriptor descriptor) {
+        if (properties.keySet().contains(descriptor)) {
+            return super.getProperty(descriptor);
+        }
+        return null;
+    }
+
+}

nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ValidationContextAdapter.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.script.impl;
+
+import java.util.Map;
+
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.controller.ControllerService;
+import org.apache.nifi.controller.ControllerServiceLookup;
+import org.apache.nifi.expression.ExpressionLanguageCompiler;
+
+public abstract class ValidationContextAdapter implements ValidationContext {
+
+    private ValidationContext innerValidationContext;
+
+    public ValidationContextAdapter(ValidationContext innerValidationContext) {
+        assert innerValidationContext != null;
+        this.innerValidationContext = innerValidationContext;
+    }
+
+    @Override
+    public PropertyValue getProperty(PropertyDescriptor descriptor) {
+        return innerValidationContext.getProperty(descriptor);
+    }
+
+    @Override
+    public Map<String, String> getAllProperties() {
+        return innerValidationContext.getAllProperties();
+    }
+
+    @Override
+    public ControllerServiceLookup getControllerServiceLookup() {
+        return innerValidationContext.getControllerServiceLookup();
+    }
+
+    @Override
+    public ValidationContext getControllerServiceValidationContext(ControllerService controllerService) {
+        return innerValidationContext.getControllerServiceValidationContext(controllerService);
+    }
+
+    @Override
+    public ExpressionLanguageCompiler newExpressionLanguageCompiler() {
+        return innerValidationContext.newExpressionLanguageCompiler();
+    }
+
+    @Override
+    public PropertyValue newPropertyValue(String value) {
+        return innerValidationContext.newPropertyValue(value);
+    }
+
+    @Override
+    public Map<PropertyDescriptor, String> getProperties() {
+        return innerValidationContext.getProperties();
+    }
+
+    @Override
+    public String getAnnotationData() {
+        return innerValidationContext.getAnnotationData();
+    }
+
+    @Override
+    public boolean isValidationRequired(ControllerService service) {
+        return innerValidationContext.isValidationRequired(service);
+    }
+
+    @Override
+    public boolean isExpressionLanguagePresent(String value) {
+        return innerValidationContext.isExpressionLanguagePresent(value);
+    }
+
+    @Override
+    public boolean isExpressionLanguageSupported(String propertyName) {
+        return innerValidationContext.isExpressionLanguageSupported(propertyName);
+    }
+
+    @Override
+    public String getProcessGroupIdentifier() {
+        return innerValidationContext.getProcessGroupIdentifier();
+    }
+
+}

nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java

@@ -16,6 +16,8 @@
  */
 package org.apache.nifi.processors.script;
 
+import org.apache.commons.codec.binary.Hex;
+
 import org.apache.nifi.components.PropertyDescriptor;
 import org.apache.nifi.processor.Relationship;
 import org.apache.nifi.script.ScriptingComponentUtils;
@@ -29,6 +31,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
 import java.util.List;
 import java.util.Set;
 
@@ -36,6 +39,7 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
+
 public class TestInvokeGroovy extends BaseScriptTest {
 
     @Before
@@ -179,4 +183,30 @@ public class TestInvokeGroovy extends BaseScriptTest {
         runner.setProperty("test-attribute", "test");
         runner.assertValid();
     }
+
+    /**
+     * Tests a script that derive from AbstractProcessor as base class
+     *
+     * @throws Exception Any error encountered while testing
+     */
+    @Test
+    public void testAbstractProcessorImplementationWithBodyScriptFile() throws Exception {
+        runner.setValidateExpressionUsage(false);
+        runner.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE, "Groovy");
+        runner.setProperty(ScriptingComponentUtils.SCRIPT_BODY, getFileContentsAsString(TEST_RESOURCE_LOCATION + "groovy/test_implementingabstractProcessor.groovy"));
+        runner.setProperty(ScriptingComponentUtils.MODULES, TEST_RESOURCE_LOCATION + "groovy");
+        runner.setProperty("custom_prop", "bla bla");
+
+        runner.assertValid();
+        runner.enqueue("test".getBytes(StandardCharsets.UTF_8));
+        runner.run();
+
+        runner.assertAllFlowFilesTransferred("success", 1);
+        final List<MockFlowFile> result = runner.getFlowFilesForRelationship("success");
+        assertTrue(result.size() == 1);
+        final String expectedOutput = new String(Hex.encodeHex(MessageDigest.getInstance("MD5").digest("testbla bla".getBytes())));
+        final MockFlowFile outputFlowFile = result.get(0);
+        outputFlowFile.assertContentEquals(expectedOutput);
+        outputFlowFile.assertAttributeEquals("outAttr", expectedOutput);
+    }
 }

nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_implementingabstractProcessor.groovy

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.commons.codec.binary.Hex
+
+import org.apache.nifi.flowfile.FlowFile
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult
+import org.apache.nifi.processor.ProcessContext
+import org.apache.nifi.processor.ProcessSession
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException
+import org.apache.nifi.processor.io.InputStreamCallback
+import org.apache.nifi.processor.io.OutputStreamCallback
+import org.apache.nifi.processor.util.StandardValidators
+import org.apache.nifi.stream.io.StreamUtils
+
+import java.security.MessageDigest
+
+
+class TestAbstractProcessor extends AbstractProcessor {
+
+	def REL_SUCCESS = new Relationship.Builder()
+			.name("success")
+			.description("FlowFiles that were successfully processed")
+			.build();
+
+	final static String attr = "outAttr";
+
+	final static PropertyDescriptor myCustomProp = new PropertyDescriptor.Builder()
+			.name("custom_prop")
+			.displayName("a custom prop")
+			.description("bla bla bla")
+			.expressionLanguageSupported(false)
+			.required(true)
+			.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+			.build();
+
+	@Override
+	List<PropertyDescriptor> getSupportedPropertyDescriptors(){
+		return [myCustomProp] as List
+	}
+
+	@Override
+	Set<Relationship> getRelationships() {
+		return [REL_SUCCESS] as Set
+	}
+
+	@Override
+	Collection<ValidationResult> customValidate(ValidationContext context) {
+		final String myCustomPropValue = context.getProperty(myCustomProp).getValue()
+		if (myCustomPropValue.length() < 3) {
+			final List<ValidationResult> problems = new ArrayList<>(1);
+			problems.add(new ValidationResult.Builder().subject(myCustomProp.getName())
+					.input(myCustomPropValue)
+					.valid(false)
+					.explanation("myCustomProp should contain a string with at least 3 chars.")
+					.build());
+			return problems;
+		}
+		return super.customValidate(context)
+	}
+
+	@Override
+	void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
+		FlowFile requestFlowFile = session.get();
+		String myCustomPropValue = context.getProperty(myCustomProp).evaluateAttributeExpressions(requestFlowFile).getValue();
+		final byte[] buff = new byte[(int) requestFlowFile.getSize()];
+		session.read(requestFlowFile, new InputStreamCallback() {
+			@Override
+			void process(InputStream inp) throws IOException {
+				StreamUtils.fillBuffer(inp, buff);
+			}
+		});
+		final String content = new String(buff);
+		final String md5 = generateMD5_A(content + myCustomPropValue);
+		requestFlowFile = session.putAttribute(requestFlowFile, attr, md5);
+		requestFlowFile = session.write(requestFlowFile, new OutputStreamCallback() {
+			@Override
+			public void process(OutputStream out) throws IOException {
+				out.write(md5.getBytes());
+			}
+		});
+
+		session.transfer(requestFlowFile, REL_SUCCESS);
+	}
+
+    static def generateMD5_A(String s){
+        new String(Hex.encodeHex(MessageDigest.getInstance("MD5").digest(s.bytes)));
+	}
+}
+
+processor = new TestAbstractProcessor();