Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-02-09 11:35:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-4130 Add lookup controller service in TransformXML to define XSLT from the UI

Browse Source 
addressed review comments

Signed-off-by: Matthew Burgess <mattyb149@apache.org>

This closes #1953
This commit is contained in:
Pierre Villard 2017-06-26 17:48:06 +02:00 committed by Matthew Burgess
parent 9e7610ac70
commit 4112af013d
2 changed files with 238 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
View File

@ -16,21 +16,24 @@
*/
package org.apache.nifi.processors.standard;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import javax.xml.XMLConstants;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Templates;
@ -39,6 +42,8 @@ import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
@ -49,6 +54,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
@ -56,6 +62,9 @@ import org.apache.nifi.expression.AttributeExpression;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.lookup.LookupFailureException;
import org.apache.nifi.lookup.LookupService;
import org.apache.nifi.lookup.StringLookupService;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
@ -67,6 +76,10 @@ import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.util.StopWatch;
import org.apache.nifi.util.Tuple;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@EventDriven
@SideEffectFree
@SupportsBatching
@ -82,12 +95,33 @@ public class TransformXml extends AbstractProcessor {
public static final PropertyDescriptor XSLT_FILE_NAME = new PropertyDescriptor.Builder()
.name("XSLT file name")
.description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content.")
.required(true)
.description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content."
+ "One of the 'XSLT file name' and 'XSLT Lookup' properties must be defined.")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
.build();
public static final PropertyDescriptor XSLT_CONTROLLER = new PropertyDescriptor.Builder()
.name("xslt-controller")
.displayName("XSLT Lookup")
.description("Controller lookup used to store XSLT definitions. One of the 'XSLT file name' and "
+ "'XSLT Lookup' properties must be defined. WARNING: note that the lookup controller service "
+ "should not be used to store large XSLT files.")
.required(false)
.identifiesControllerService(StringLookupService.class)
.build();
public static final PropertyDescriptor XSLT_CONTROLLER_KEY = new PropertyDescriptor.Builder()
.name("xslt-controller-key")
.displayName("XSLT Lookup key")
.description("Key used to retrieve the XSLT definition from the XSLT lookup controller. This property must be "
+ "set when using the XSLT controller property.")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR)
.build();
public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder()
.name("indent-output")
.displayName("Indent")
@ -140,10 +174,14 @@ public class TransformXml extends AbstractProcessor {
private Set<Relationship> relationships;
private LoadingCache<String, Templates> cache;
private static AtomicReference<LookupService<String>> lookupService = new AtomicReference<LookupService<String>>(null);
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(XSLT_FILE_NAME);
properties.add(XSLT_CONTROLLER);
properties.add(XSLT_CONTROLLER_KEY);
properties.add(INDENT_OUTPUT);
properties.add(SECURE_PROCESSING);
properties.add(CACHE_SIZE);
@ -166,6 +204,47 @@ public class TransformXml extends AbstractProcessor {
return properties;
}
@Override
protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
final List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
PropertyValue filename = validationContext.getProperty(XSLT_FILE_NAME);
PropertyValue controller = validationContext.getProperty(XSLT_CONTROLLER);
PropertyValue key = validationContext.getProperty(XSLT_CONTROLLER_KEY);
if((filename.isSet() && controller.isSet())
|| (!filename.isSet() && !controller.isSet())) {
results.add(new ValidationResult.Builder()
.valid(false)
.subject(this.getClass().getSimpleName())
.explanation("Exactly one of the \"XSLT file name\" and \"XSLT controller\" properties must be defined.")
.build());
}
if(controller.isSet() && !key.isSet()) {
results.add(new ValidationResult.Builder()
.valid(false)
.subject(XSLT_CONTROLLER_KEY.getDisplayName())
.explanation("If using \"XSLT controller\", the XSLT controller key property must be defined.")
.build());
}
if(controller.isSet()) {
final LookupService<String> lookupService = validationContext.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class);
final Set<String> requiredKeys = lookupService.getRequiredKeys();
if (requiredKeys == null || requiredKeys.size() != 1) {
results.add(new ValidationResult.Builder()
.valid(false)
.subject(XSLT_CONTROLLER.getDisplayName())
.explanation("This processor requires a key-value lookup service supporting exactly one required key, was: " +
(requiredKeys == null ? "null" : String.valueOf(requiredKeys.size())))
.build());
}
}
return results;
}
@Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
return new PropertyDescriptor.Builder()
@ -177,9 +256,10 @@ public class TransformXml extends AbstractProcessor {
.build();
}
private Templates newTemplates(ProcessContext context, String path) throws TransformerConfigurationException {
private Templates newTemplates(final ProcessContext context, final String path) throws TransformerConfigurationException, LookupFailureException {
final Boolean secureProcessing = context.getProperty(SECURE_PROCESSING).asBoolean();
TransformerFactory factory = TransformerFactory.newInstance();
final boolean isFilename = context.getProperty(XSLT_FILE_NAME).isSet();
if (secureProcessing) {
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
@ -188,7 +268,17 @@ public class TransformXml extends AbstractProcessor {
factory.setFeature("http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities", false);
}
return factory.newTemplates(new StreamSource(path));
if(isFilename) {
return factory.newTemplates(new StreamSource(path));
} else {
final String coordinateKey = lookupService.get().getRequiredKeys().iterator().next();
final Optional<String> attributeValue = lookupService.get().lookup(Collections.singletonMap(coordinateKey, path));
if (attributeValue.isPresent() && StringUtils.isNotBlank(attributeValue.get())) {
return factory.newTemplates(new StreamSource(new ByteArrayInputStream(attributeValue.get().getBytes(StandardCharsets.UTF_8))));
} else {
throw new TransformerConfigurationException("No XSLT definition is associated to " + path + " in the lookup controller service.");
}
}
}
@OnScheduled
@ -198,20 +288,21 @@ public class TransformXml extends AbstractProcessor {
final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS);
if (cacheSize > 0) {
CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize);
if (cacheTTL > 0) {
cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS);
}
cache = cacheBuilder.build(
new CacheLoader<String, Templates>() {
public Templates load(String path) throws TransformerConfigurationException {
return newTemplates(context, path);
}
});
new CacheLoader<String, Templates>() {
@Override
public Templates load(String path) throws TransformerConfigurationException, LookupFailureException {
return newTemplates(context, path);
}
});
} else {
cache = null;
logger.warn("Stylesheet cache disabled because cache size is set to 0");
logger.info("Stylesheet cache disabled because cache size is set to 0");
}
}
@ -224,10 +315,11 @@ public class TransformXml extends AbstractProcessor {
final ComponentLog logger = getLogger();
final StopWatch stopWatch = new StopWatch(true);
final String xsltFileName = context.getProperty(XSLT_FILE_NAME)
.evaluateAttributeExpressions(original)
.getValue();
final String path = context.getProperty(XSLT_FILE_NAME).isSet()
? context.getProperty(XSLT_FILE_NAME).evaluateAttributeExpressions(original).getValue()
: context.getProperty(XSLT_CONTROLLER_KEY).evaluateAttributeExpressions(original).getValue();
final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean();
lookupService.set(context.getProperty(XSLT_CONTROLLER).asControllerService(LookupService.class));
try {
FlowFile transformed = session.write(original, new StreamCallback() {
@ -236,9 +328,9 @@ public class TransformXml extends AbstractProcessor {
try (final InputStream in = new BufferedInputStream(rawIn)) {
final Templates templates;
if (cache != null) {
templates = cache.get(xsltFileName);
templates = cache.get(path);
} else {
templates = newTemplates(context, xsltFileName);
templates = newTemplates(context, path);
}
final Transformer transformer = templates.newTransformer();
@ -303,4 +395,4 @@ public class TransformXml extends AbstractProcessor {
}
}
}
}

133
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java
View File

@ -16,23 +16,21 @@
*/
package org.apache.nifi.processors.standard;
import java.io.IOException;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import org.apache.nifi.lookup.SimpleKeyValueLookupService;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.Test;
public class TestTransformXml {
@ -56,8 +54,6 @@ public class TestTransformXml {
runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
final MockFlowFile original = runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0);
final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8);
original.assertContentEquals("not xml");
}
@ -107,7 +103,6 @@ public class TestTransformXml {
runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1);
final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/tokens.xml")));
transformed.assertContentEquals(expectedContent);
@ -148,4 +143,128 @@ public class TestTransformXml {
transformed.assertContentEquals(expectedContent);
}
@Test
public void testTransformBothControllerFileNotValid() throws IOException, InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl");
final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
runner.addControllerService("simple-key-value-lookup-service", service);
runner.setProperty(service, "key1", "value1");
runner.enableControllerService(service);
runner.assertValid(service);
runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
runner.assertNotValid();
}
@Test
public void testTransformNoneControllerFileNotValid() throws IOException, InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
runner.setProperty(TransformXml.CACHE_SIZE, "0");
runner.assertNotValid();
}
@Test
public void testTransformControllerNoKey() throws IOException, InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
runner.addControllerService("simple-key-value-lookup-service", service);
runner.setProperty(service, "key1", "value1");
runner.enableControllerService(service);
runner.assertValid(service);
runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
runner.assertNotValid();
}
@Test
public void testTransformWithController() throws IOException, InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
runner.addControllerService("simple-key-value-lookup-service", service);
runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">"
+ "<xsl:param name=\"header\" /><xsl:template match=\"doc\">"
+ "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>"
+ "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>"
+ "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>"
+ "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>"
+ "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>"
+ "</HTML></xsl:template></xsl:stylesheet>");
runner.enableControllerService(service);
runner.assertValid(service);
runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
runner.setProperty("header", "Test for mod");
final Map<String, String> attributes = new HashMap<>();
attributes.put("xslt", "math");
runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
runner.run();
runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim();
transformed.assertContentEquals(expectedContent);
}
@Test
public void testTransformWithXsltNotFoundInController() throws IOException, InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
runner.addControllerService("simple-key-value-lookup-service", service);
runner.enableControllerService(service);
runner.assertValid(service);
runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
runner.setProperty("header", "Test for mod");
final Map<String, String> attributes = new HashMap<>();
attributes.put("xslt", "math");
runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
runner.run();
runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
}
@Test
public void testTransformWithControllerNoCache() throws IOException, InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService();
runner.addControllerService("simple-key-value-lookup-service", service);
runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns:xs=\"http://www.w3.org/2001/XMLSchema\">"
+ "<xsl:param name=\"header\" /><xsl:template match=\"doc\">"
+ "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>"
+ "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>"
+ "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>"
+ "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>"
+ "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>"
+ "</HTML></xsl:template></xsl:stylesheet>");
runner.enableControllerService(service);
runner.assertValid(service);
runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service");
runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}");
runner.setProperty(TransformXml.CACHE_SIZE, "0");
runner.setProperty("header", "Test for mod");
final Map<String, String> attributes = new HashMap<>();
attributes.put("xslt", "math");
runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
runner.run();
runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim();
transformed.assertContentEquals(expectedContent);
}
}