NIFI-4130 Add lookup controller service in TransformXML to define XSLT from the UI

addressed review comments

Signed-off-by: Matthew Burgess <mattyb149@apache.org>

This closes #1953

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java

@@ -16,21 +16,24 @@
  */
 package org.apache.nifi.processors.standard;
 
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
 import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
 import javax.xml.XMLConstants;
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Templates;
@@ -39,6 +42,8 @@ import javax.xml.transform.TransformerConfigurationException;
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.stream.StreamResult;
 import javax.xml.transform.stream.StreamSource;
+
+import org.apache.commons.lang3.StringUtils;
 import org.apache.nifi.annotation.behavior.DynamicProperty;
 import org.apache.nifi.annotation.behavior.EventDriven;
 import org.apache.nifi.annotation.behavior.InputRequirement;
@@ -49,6 +54,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
 import org.apache.nifi.annotation.documentation.Tags;
 import org.apache.nifi.annotation.lifecycle.OnScheduled;
 import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
 import org.apache.nifi.components.ValidationContext;
 import org.apache.nifi.components.ValidationResult;
 import org.apache.nifi.components.Validator;
@@ -56,6 +62,9 @@ import org.apache.nifi.expression.AttributeExpression;
 import org.apache.nifi.expression.ExpressionLanguageScope;
 import org.apache.nifi.flowfile.FlowFile;
 import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.lookup.LookupFailureException;
+import org.apache.nifi.lookup.LookupService;
+import org.apache.nifi.lookup.StringLookupService;
 import org.apache.nifi.processor.AbstractProcessor;
 import org.apache.nifi.processor.ProcessContext;
 import org.apache.nifi.processor.ProcessSession;
@@ -67,6 +76,10 @@ import org.apache.nifi.processor.util.StandardValidators;
 import org.apache.nifi.util.StopWatch;
 import org.apache.nifi.util.Tuple;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
 @EventDriven
 @SideEffectFree
 @SupportsBatching
@@ -82,12 +95,33 @@ public class TransformXml extends AbstractProcessor {
 
     public static final PropertyDescriptor XSLT_FILE_NAME = new PropertyDescriptor.Builder()
             .name("XSLT file name")
-            .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content.")
+            .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content."
-            .required(true)
+                    + "One of the 'XSLT file name' and 'XSLT Lookup' properties must be defined.")
+            .required(false)
             .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
             .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
             .build();
 
+    public static final PropertyDescriptor XSLT_CONTROLLER = new PropertyDescriptor.Builder()
+            .name("xslt-controller")
+            .displayName("XSLT Lookup")
+            .description("Controller lookup used to store XSLT definitions. One of the 'XSLT file name' and "
+                    + "'XSLT Lookup' properties must be defined. WARNING: note that the lookup controller service "
+                    + "should not be used to store large XSLT files.")
+            .required(false)
+            .identifiesControllerService(StringLookupService.class)
+            .build();
+
+    public static final PropertyDescriptor XSLT_CONTROLLER_KEY = new PropertyDescriptor.Builder()
+            .name("xslt-controller-key")
+            .displayName("XSLT Lookup key")
+            .description("Key used to retrieve the XSLT definition from the XSLT lookup controller. This property must be "
+                    + "set when using the XSLT controller property.")
+            .required(false)
+            .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+            .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
+            .build();
+
     public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder()
             .name("indent-output")
             .displayName("Indent")
@@ -140,10 +174,14 @@ public class TransformXml extends AbstractProcessor {
     private Set<Relationship> relationships;
     private LoadingCache<String, Templates> cache;
 
+    private static AtomicReference<LookupService<String>> lookupService = new AtomicReference<LookupService<String>>(null);
+
     @Override
     protected void init(final ProcessorInitializationContext context) {
         final List<PropertyDescriptor> properties = new ArrayList<>();
         properties.add(XSLT_FILE_NAME);
+        properties.add(XSLT_CONTROLLER);
+        properties.add(XSLT_CONTROLLER_KEY);
         properties.add(INDENT_OUTPUT);
         properties.add(SECURE_PROCESSING);
         properties.add(CACHE_SIZE);
@@ -166,6 +204,47 @@ public class TransformXml extends AbstractProcessor {
         return properties;
     }
 
+    @Override
+    protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+        final List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+
+        PropertyValue filename = validationContext.getProperty(XSLT_FILE_NAME);
+        PropertyValue controller = validationContext.getProperty(XSLT_CONTROLLER);
+        PropertyValue key = validationContext.getProperty(XSLT_CONTROLLER_KEY);
+
+        if((filename.isSet() && controller.isSet())
+                || (!filename.isSet() && !controller.isSet())) {
+            results.add(new ValidationResult.Builder()
+                    .valid(false)
+                    .subject(this.getClass().getSimpleName())
+                    .explanation("Exactly one of the \"XSLT file name\" and \"XSLT controller\" properties must be defined.")
+                    .build());
+        }
+
+        if(controller.isSet() && !key.isSet()) {
+            results.add(new ValidationResult.Builder()
+                    .valid(false)
+                    .subject(XSLT_CONTROLLER_KEY.getDisplayName())
+                    .explanation("If using \"XSLT controller\", the XSLT controller key property must be defined.")
+                    .build());
+        }
+
+        if(controller.isSet()) {
+            final LookupService<String> lookupService = validationContext.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class);
+            final Set<String> requiredKeys = lookupService.getRequiredKeys();
+            if (requiredKeys == null || requiredKeys.size() != 1) {
+                results.add(new ValidationResult.Builder()
+                        .valid(false)
+                        .subject(XSLT_CONTROLLER.getDisplayName())
+                        .explanation("This processor requires a key-value lookup service supporting exactly one required key, was: " +
+                            (requiredKeys == null ? "null" : String.valueOf(requiredKeys.size())))
+                        .build());
+            }
+        }
+
+        return results;
+    }
+
     @Override
     protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
         return new PropertyDescriptor.Builder()
@@ -177,9 +256,10 @@ public class TransformXml extends AbstractProcessor {
                 .build();
     }
 
-    private Templates newTemplates(ProcessContext context, String path) throws TransformerConfigurationException {
+    private Templates newTemplates(final ProcessContext context, final String path) throws TransformerConfigurationException, LookupFailureException {
         final Boolean secureProcessing = context.getProperty(SECURE_PROCESSING).asBoolean();
         TransformerFactory factory = TransformerFactory.newInstance();
+        final boolean isFilename = context.getProperty(XSLT_FILE_NAME).isSet();
 
         if (secureProcessing) {
             factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
@@ -188,7 +268,17 @@ public class TransformXml extends AbstractProcessor {
             factory.setFeature("http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities", false);
         }
 
-        return factory.newTemplates(new StreamSource(path));
+        if(isFilename) {
+            return factory.newTemplates(new StreamSource(path));
+        } else {
+            final String coordinateKey = lookupService.get().getRequiredKeys().iterator().next();
+            final Optional<String> attributeValue = lookupService.get().lookup(Collections.singletonMap(coordinateKey, path));
+            if (attributeValue.isPresent() && StringUtils.isNotBlank(attributeValue.get())) {
+                return factory.newTemplates(new StreamSource(new ByteArrayInputStream(attributeValue.get().getBytes(StandardCharsets.UTF_8))));
+            } else {
+                throw new TransformerConfigurationException("No XSLT definition is associated to " + path + " in the lookup controller service.");
+            }
+        }
     }
 
     @OnScheduled
@@ -198,20 +288,21 @@ public class TransformXml extends AbstractProcessor {
         final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS);
 
         if (cacheSize > 0) {
-            CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
+            CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
             if (cacheTTL > 0) {
                 cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS);
             }
 
             cache = cacheBuilder.build(
-               new CacheLoader<String, Templates>() {
+                    new CacheLoader<String, Templates>() {
-                   public Templates load(String path) throws TransformerConfigurationException {
+                        @Override
-                       return newTemplates(context, path);
+                        public Templates load(String path) throws TransformerConfigurationException, LookupFailureException {
-                   }
+                            return newTemplates(context, path);
-               });
+                        }
+                    });
         } else {
             cache = null;
-            logger.warn("Stylesheet cache disabled because cache size is set to 0");
+            logger.info("Stylesheet cache disabled because cache size is set to 0");
         }
     }
 
@@ -224,10 +315,11 @@ public class TransformXml extends AbstractProcessor {
 
         final ComponentLog logger = getLogger();
         final StopWatch stopWatch = new StopWatch(true);
-        final String xsltFileName = context.getProperty(XSLT_FILE_NAME)
+        final String path = context.getProperty(XSLT_FILE_NAME).isSet()
-            .evaluateAttributeExpressions(original)
+                ? context.getProperty(XSLT_FILE_NAME).evaluateAttributeExpressions(original).getValue()
-            .getValue();
+                        : context.getProperty(XSLT_CONTROLLER_KEY).evaluateAttributeExpressions(original).getValue();
         final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean();
+        lookupService.set(context.getProperty(XSLT_CONTROLLER).asControllerService(LookupService.class));
 
         try {
             FlowFile transformed = session.write(original, new StreamCallback() {
@@ -236,9 +328,9 @@ public class TransformXml extends AbstractProcessor {
                     try (final InputStream in = new BufferedInputStream(rawIn)) {
                         final Templates templates;
                         if (cache != null) {
-                            templates = cache.get(xsltFileName);
+                            templates = cache.get(path);
                         } else {
-                            templates = newTemplates(context, xsltFileName);
+                            templates = newTemplates(context, path);
                         }
 
                         final Transformer transformer = templates.newTransformer();
@@ -303,4 +395,4 @@ public class TransformXml extends AbstractProcessor {
         }
     }
 
-}
+}

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java

@@ -16,23 +16,21 @@
  */
 package org.apache.nifi.processors.standard;
 
-import java.io.IOException;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.nifi.lookup.SimpleKeyValueLookupService;
+import org.apache.nifi.reporting.InitializationException;
 import org.apache.nifi.util.MockFlowFile;
 import org.apache.nifi.util.TestRunner;
 import org.apache.nifi.util.TestRunners;
-
 import org.junit.Test;
 
 public class TestTransformXml {
@@ -56,8 +54,6 @@ public class TestTransformXml {
 
         runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
         final MockFlowFile original = runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0);
-        final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8);
-
         original.assertContentEquals("not xml");
     }
 
@@ -107,7 +103,6 @@ public class TestTransformXml {
 
             runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
             final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
-            final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1);
             final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/tokens.xml")));
 
             transformed.assertContentEquals(expectedContent);
@@ -148,4 +143,128 @@ public class TestTransformXml {
         transformed.assertContentEquals(expectedContent);
     }
 
+    @Test
+    public void testTransformBothControllerFileNotValid() throws IOException, InitializationException {
+        final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+        runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl");
+
+        final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+        runner.addControllerService("simple-key-value-lookup-service", service);
+        runner.setProperty(service, "key1", "value1");
+        runner.enableControllerService(service);
+        runner.assertValid(service);
+        runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+
+        runner.assertNotValid();
+    }
+
+    @Test
+    public void testTransformNoneControllerFileNotValid() throws IOException, InitializationException {
+        final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+        runner.setProperty(TransformXml.CACHE_SIZE, "0");
+        runner.assertNotValid();
+    }
+
+    @Test
+    public void testTransformControllerNoKey() throws IOException, InitializationException {
+        final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+        final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+        runner.addControllerService("simple-key-value-lookup-service", service);
+        runner.setProperty(service, "key1", "value1");
+        runner.enableControllerService(service);
+        runner.assertValid(service);
+        runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+
+        runner.assertNotValid();
+    }
+
+    @Test
+    public void testTransformWithController() throws IOException, InitializationException {
+        final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+        final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+        runner.addControllerService("simple-key-value-lookup-service", service);
+        runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">"
+                + "<xsl:param name=\"header\" /><xsl:template match=\"doc\">"
+                + "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>"
+                + "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>"
+                + "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>"
+                + "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>"
+                + "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>"
+                + "</HTML></xsl:template></xsl:stylesheet>");
+        runner.enableControllerService(service);
+        runner.assertValid(service);
+        runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+        runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
+        runner.setProperty("header", "Test for mod");
+
+        final Map<String, String> attributes = new HashMap<>();
+        attributes.put("xslt", "math");
+        runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
+
+        runner.run();
+
+        runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
+        final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
+        final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim();
+
+        transformed.assertContentEquals(expectedContent);
+    }
+
+    @Test
+    public void testTransformWithXsltNotFoundInController() throws IOException, InitializationException {
+        final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+        final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+        runner.addControllerService("simple-key-value-lookup-service", service);
+        runner.enableControllerService(service);
+        runner.assertValid(service);
+        runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+        runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
+        runner.setProperty("header", "Test for mod");
+
+        final Map<String, String> attributes = new HashMap<>();
+        attributes.put("xslt", "math");
+        runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
+
+        runner.run();
+
+        runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
+    }
+
+    @Test
+    public void testTransformWithControllerNoCache() throws IOException, InitializationException {
+        final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
+
+        final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
+        runner.addControllerService("simple-key-value-lookup-service", service);
+        runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">"
+                + "<xsl:param name=\"header\" /><xsl:template match=\"doc\">"
+                + "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>"
+                + "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>"
+                + "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>"
+                + "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>"
+                + "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>"
+                + "</HTML></xsl:template></xsl:stylesheet>");
+        runner.enableControllerService(service);
+        runner.assertValid(service);
+        runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
+        runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
+        runner.setProperty(TransformXml.CACHE_SIZE, "0");
+        runner.setProperty("header", "Test for mod");
+
+        final Map<String, String> attributes = new HashMap<>();
+        attributes.put("xslt", "math");
+        runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
+
+        runner.run();
+
+        runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
+        final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
+        final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim();
+
+        transformed.assertContentEquals(expectedContent);
+    }
+
 }