From 4112af013d1b1f49e83f881d85ebe66e097840b5 Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Mon, 26 Jun 2017 17:48:06 +0200 Subject: [PATCH] NIFI-4130 Add lookup controller service in TransformXML to define XSLT from the UI addressed review comments Signed-off-by: Matthew Burgess This closes #1953 --- .../processors/standard/TransformXml.java | 132 ++++++++++++++--- .../processors/standard/TestTransformXml.java | 133 +++++++++++++++++- 2 files changed, 238 insertions(+), 27 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java index ff15428adc..1cc57fae24 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java @@ -16,21 +16,24 @@ */ package org.apache.nifi.processors.standard; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + import javax.xml.XMLConstants; import javax.xml.transform.OutputKeys; import javax.xml.transform.Templates; @@ -39,6 +42,8 @@ import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; + +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.InputRequirement; @@ -49,6 +54,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; @@ -56,6 +62,9 @@ import org.apache.nifi.expression.AttributeExpression; import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.lookup.LookupFailureException; +import org.apache.nifi.lookup.LookupService; +import org.apache.nifi.lookup.StringLookupService; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -67,6 +76,10 @@ import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.util.StopWatch; import org.apache.nifi.util.Tuple; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + @EventDriven @SideEffectFree @SupportsBatching @@ -82,12 +95,33 @@ public class TransformXml extends AbstractProcessor { public static final PropertyDescriptor XSLT_FILE_NAME = new PropertyDescriptor.Builder() .name("XSLT file name") - .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content.") - .required(true) + .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content." + + "One of the 'XSLT file name' and 'XSLT Lookup' properties must be defined.") + .required(false) .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) .build(); + public static final PropertyDescriptor XSLT_CONTROLLER = new PropertyDescriptor.Builder() + .name("xslt-controller") + .displayName("XSLT Lookup") + .description("Controller lookup used to store XSLT definitions. One of the 'XSLT file name' and " + + "'XSLT Lookup' properties must be defined. WARNING: note that the lookup controller service " + + "should not be used to store large XSLT files.") + .required(false) + .identifiesControllerService(StringLookupService.class) + .build(); + + public static final PropertyDescriptor XSLT_CONTROLLER_KEY = new PropertyDescriptor.Builder() + .name("xslt-controller-key") + .displayName("XSLT Lookup key") + .description("Key used to retrieve the XSLT definition from the XSLT lookup controller. This property must be " + + "set when using the XSLT controller property.") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) + .build(); + public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder() .name("indent-output") .displayName("Indent") @@ -140,10 +174,14 @@ public class TransformXml extends AbstractProcessor { private Set relationships; private LoadingCache cache; + private static AtomicReference> lookupService = new AtomicReference>(null); + @Override protected void init(final ProcessorInitializationContext context) { final List properties = new ArrayList<>(); properties.add(XSLT_FILE_NAME); + properties.add(XSLT_CONTROLLER); + properties.add(XSLT_CONTROLLER_KEY); properties.add(INDENT_OUTPUT); properties.add(SECURE_PROCESSING); properties.add(CACHE_SIZE); @@ -166,6 +204,47 @@ public class TransformXml extends AbstractProcessor { return properties; } + @Override + protected Collection customValidate(ValidationContext validationContext) { + final List results = new ArrayList<>(super.customValidate(validationContext)); + + PropertyValue filename = validationContext.getProperty(XSLT_FILE_NAME); + PropertyValue controller = validationContext.getProperty(XSLT_CONTROLLER); + PropertyValue key = validationContext.getProperty(XSLT_CONTROLLER_KEY); + + if((filename.isSet() && controller.isSet()) + || (!filename.isSet() && !controller.isSet())) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(this.getClass().getSimpleName()) + .explanation("Exactly one of the \"XSLT file name\" and \"XSLT controller\" properties must be defined.") + .build()); + } + + if(controller.isSet() && !key.isSet()) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(XSLT_CONTROLLER_KEY.getDisplayName()) + .explanation("If using \"XSLT controller\", the XSLT controller key property must be defined.") + .build()); + } + + if(controller.isSet()) { + final LookupService lookupService = validationContext.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class); + final Set requiredKeys = lookupService.getRequiredKeys(); + if (requiredKeys == null || requiredKeys.size() != 1) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(XSLT_CONTROLLER.getDisplayName()) + .explanation("This processor requires a key-value lookup service supporting exactly one required key, was: " + + (requiredKeys == null ? "null" : String.valueOf(requiredKeys.size()))) + .build()); + } + } + + return results; + } + @Override protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { return new PropertyDescriptor.Builder() @@ -177,9 +256,10 @@ public class TransformXml extends AbstractProcessor { .build(); } - private Templates newTemplates(ProcessContext context, String path) throws TransformerConfigurationException { + private Templates newTemplates(final ProcessContext context, final String path) throws TransformerConfigurationException, LookupFailureException { final Boolean secureProcessing = context.getProperty(SECURE_PROCESSING).asBoolean(); TransformerFactory factory = TransformerFactory.newInstance(); + final boolean isFilename = context.getProperty(XSLT_FILE_NAME).isSet(); if (secureProcessing) { factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -188,7 +268,17 @@ public class TransformXml extends AbstractProcessor { factory.setFeature("http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities", false); } - return factory.newTemplates(new StreamSource(path)); + if(isFilename) { + return factory.newTemplates(new StreamSource(path)); + } else { + final String coordinateKey = lookupService.get().getRequiredKeys().iterator().next(); + final Optional attributeValue = lookupService.get().lookup(Collections.singletonMap(coordinateKey, path)); + if (attributeValue.isPresent() && StringUtils.isNotBlank(attributeValue.get())) { + return factory.newTemplates(new StreamSource(new ByteArrayInputStream(attributeValue.get().getBytes(StandardCharsets.UTF_8)))); + } else { + throw new TransformerConfigurationException("No XSLT definition is associated to " + path + " in the lookup controller service."); + } + } } @OnScheduled @@ -198,20 +288,21 @@ public class TransformXml extends AbstractProcessor { final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS); if (cacheSize > 0) { - CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize); + CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize); if (cacheTTL > 0) { cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS); } cache = cacheBuilder.build( - new CacheLoader() { - public Templates load(String path) throws TransformerConfigurationException { - return newTemplates(context, path); - } - }); + new CacheLoader() { + @Override + public Templates load(String path) throws TransformerConfigurationException, LookupFailureException { + return newTemplates(context, path); + } + }); } else { cache = null; - logger.warn("Stylesheet cache disabled because cache size is set to 0"); + logger.info("Stylesheet cache disabled because cache size is set to 0"); } } @@ -224,10 +315,11 @@ public class TransformXml extends AbstractProcessor { final ComponentLog logger = getLogger(); final StopWatch stopWatch = new StopWatch(true); - final String xsltFileName = context.getProperty(XSLT_FILE_NAME) - .evaluateAttributeExpressions(original) - .getValue(); + final String path = context.getProperty(XSLT_FILE_NAME).isSet() + ? context.getProperty(XSLT_FILE_NAME).evaluateAttributeExpressions(original).getValue() + : context.getProperty(XSLT_CONTROLLER_KEY).evaluateAttributeExpressions(original).getValue(); final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean(); + lookupService.set(context.getProperty(XSLT_CONTROLLER).asControllerService(LookupService.class)); try { FlowFile transformed = session.write(original, new StreamCallback() { @@ -236,9 +328,9 @@ public class TransformXml extends AbstractProcessor { try (final InputStream in = new BufferedInputStream(rawIn)) { final Templates templates; if (cache != null) { - templates = cache.get(xsltFileName); + templates = cache.get(path); } else { - templates = newTemplates(context, xsltFileName); + templates = newTemplates(context, path); } final Transformer transformer = templates.newTransformer(); @@ -303,4 +395,4 @@ public class TransformXml extends AbstractProcessor { } } -} +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java index 5ced26657d..3d5f1e0bf7 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java @@ -16,23 +16,21 @@ */ package org.apache.nifi.processors.standard; -import java.io.IOException; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import org.apache.nifi.lookup.SimpleKeyValueLookupService; +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; - import org.junit.Test; public class TestTransformXml { @@ -56,8 +54,6 @@ public class TestTransformXml { runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE); final MockFlowFile original = runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0); - final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8); - original.assertContentEquals("not xml"); } @@ -107,7 +103,6 @@ public class TestTransformXml { runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); - final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1); final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/tokens.xml"))); transformed.assertContentEquals(expectedContent); @@ -148,4 +143,128 @@ public class TestTransformXml { transformed.assertContentEquals(expectedContent); } + @Test + public void testTransformBothControllerFileNotValid() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl"); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "key1", "value1"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + + runner.assertNotValid(); + } + + @Test + public void testTransformNoneControllerFileNotValid() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + runner.setProperty(TransformXml.CACHE_SIZE, "0"); + runner.assertNotValid(); + } + + @Test + public void testTransformControllerNoKey() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "key1", "value1"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + + runner.assertNotValid(); + } + + @Test + public void testTransformWithController() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "math", "" + + "" + + "

" + + "

Should say \"1\":

" + + "

Should say \"1\":

" + + "

Should say \"-1\":

" + + "

" + + ""); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty("header", "Test for mod"); + + final Map attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); + + transformed.assertContentEquals(expectedContent); + } + + @Test + public void testTransformWithXsltNotFoundInController() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty("header", "Test for mod"); + + final Map attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE); + } + + @Test + public void testTransformWithControllerNoCache() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "math", "" + + "" + + "

" + + "

Should say \"1\":

" + + "

Should say \"1\":

" + + "

Should say \"-1\":

" + + "

" + + ""); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty(TransformXml.CACHE_SIZE, "0"); + runner.setProperty("header", "Test for mod"); + + final Map attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); + + transformed.assertContentEquals(expectedContent); + } + }