Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9901 Added nifi-xml-processing to nifi-commons 

- Refactored XML parsing to use providers from nifi-xml-processing
- Configured spotbugs-maven-plugin with findsecbugs-plugin in nifi-xml-processing
- Disabled Validate DTD in default configuration for EvaluateXPath and EvaluateXQuery
- Replaced configuration of DocumentBuilder and streaming XML Readers with shared components
- Removed XML utilities from nifi-security-utils
- Moved Commons Configuration classes to nifi-lookup-services

This closes #5962
Signed-off-by: Paul Grey <greyp@apache.org>
This commit is contained in:
exceptionfactory 2022-04-12 15:32:27 -05:00 committed by Paul Grey
parent 05f3d7510f
commit 15f7590f7a
No known key found for this signature in database
GPG Key ID: 8DDF32B9C7EE39D0
120 changed files with 2081 additions and 1267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
minifi/minifi-bootstrap/pom.xml
View File

@ -56,6 +56,10 @@ limitations under the License.
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>

19
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
View File

@ -46,15 +46,14 @@ import org.apache.nifi.minifi.commons.schema.common.Schema;
import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
@ -148,7 +147,7 @@ public final class ConfigTransformer {
}
}
protected static void writeFlowXmlFile(ConfigSchema configSchema, OutputStream outputStream) throws TransformerException, ConfigTransformerException, ConfigurationChangeException, IOException {
protected static void writeFlowXmlFile(ConfigSchema configSchema, OutputStream outputStream) throws TransformerException, ConfigTransformerException {
final StreamResult streamResult = new StreamResult(outputStream);
// configure the transformer and convert the DOM
@ -307,14 +306,12 @@ public final class ConfigTransformer {
}
}
protected static DOMSource createFlowXml(ConfigSchema configSchema) throws IOException, ConfigurationChangeException, ConfigTransformerException {
protected static DOMSource createFlowXml(ConfigSchema configSchema) throws ConfigTransformerException {
try {
// create a new, empty document
final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setNamespaceAware(true);
final DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
final Document doc = docBuilder.newDocument();
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
final Document doc = documentProvider.newDocument();
// populate document with controller state
final Element rootNode = doc.createElement("flowController");
@ -365,7 +362,7 @@ public final class ConfigTransformer {
}
return new DOMSource(doc);
} catch (final ParserConfigurationException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) {
} catch (final ProcessingException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) {
throw new ConfigTransformerException(e);
} catch (Exception e) {
throw new ConfigTransformerException("Failed to parse the config YAML while writing the top level of the flow xml", e);

39
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
View File

@ -34,6 +34,8 @@ import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
import org.apache.nifi.util.StringUtils;
import org.apache.nifi.xml.processing.parsers.DocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -42,9 +44,6 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
@ -55,7 +54,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringBufferInputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
@ -80,15 +78,14 @@ public class ConfigTransformerTest {
Arrays.asList("processor", "inputPort", "outputPort", "funnel", "processGroup", "remoteProcessGroup", "connection"));
private XPathFactory xPathFactory;
private Element config;
private DocumentBuilder documentBuilder;
@Rule
final public TemporaryFolder tempOutputFolder = new TemporaryFolder();
@Before
public void setup() throws ParserConfigurationException {
documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
final Document document = documentBuilder.newDocument();
public void setup() {
final DocumentProvider documentProvider = new StandardDocumentProvider();
final Document document = documentProvider.newDocument();
config = document.createElement("config");
xPathFactory = XPathFactory.newInstance();
}
@ -484,11 +481,11 @@ public class ConfigTransformerTest {
assertTrue(flowXml.exists());
assertTrue(flowXml.canRead());
String flow = loadFlowXML(new FileInputStream(flowXml));
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
Document xml = db.parse(new StringBufferInputStream(flow));
final DocumentProvider documentProvider = new StandardDocumentProvider();
final Document xml;
try (final InputStream inputStream = new GZIPInputStream(new FileInputStream(flowXml))) {
xml = documentProvider.parse(inputStream);
}
XPath xPath = XPathFactory.newInstance().newXPath();
String result = xPath.evaluate("/flowController/rootGroup/processor/property[name = \"SSL Context Service\"]/value/text()", xml);
@ -504,7 +501,8 @@ public class ConfigTransformerTest {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ConfigTransformer.writeFlowXmlFile(configSchema, outputStream);
Document document = documentBuilder.parse(new ByteArrayInputStream(outputStream.toByteArray()));
final DocumentProvider documentProvider = new StandardDocumentProvider();
Document document = documentProvider.parse(new ByteArrayInputStream(outputStream.toByteArray()));
testProcessGroup((Element) xPathFactory.newXPath().evaluate("flowController/rootGroup", document, XPathConstants.NODE), configSchema.getProcessGroupSchema());
testReportingTasks((Element) xPathFactory.newXPath().evaluate("flowController/reportingTasks", document, XPathConstants.NODE), configSchema.getReportingTasksSchema());
@ -786,17 +784,4 @@ public class ConfigTransformerTest {
}
return bootstrapProperties;
}
public static String loadFlowXML(InputStream compressedData) throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
GZIPInputStream gzipInputStream = new GZIPInputStream(compressedData);
byte[] buffer = new byte[1024];
int len;
while ((len = gzipInputStream.read(buffer)) != -1) {
byteArrayOutputStream.write(buffer, 0, len);
}
return byteArrayOutputStream.toString();
}
}

5
minifi/pom.xml
View File

@ -385,6 +385,11 @@ limitations under the License.
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-logging-utils</artifactId>

5
nifi-bootstrap/pom.xml
View File

@ -45,6 +45,11 @@ language governing permissions and limitations under the License. -->
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-flow-encryptor</artifactId>

36
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java
View File

@ -31,18 +31,14 @@ import org.apache.nifi.components.resource.StandardResourceContext;
import org.apache.nifi.components.resource.StandardResourceReferenceFactory;
import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.registry.VariableRegistry;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
@ -91,27 +87,6 @@ public class NotificationServiceManager {
this.maxAttempts = maxAttempts;
}
private static DocumentBuilder createSafeDocumentBuilder() throws ParserConfigurationException {
final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setNamespaceAware(false);
// These features are used to disable processing external entities in the DocumentBuilderFactory to protect against XXE attacks
final String DISALLOW_DOCTYPES = "http://apache.org/xml/features/disallow-doctype-decl";
final String ALLOW_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities";
final String ALLOW_EXTERNAL_PARAM_ENTITIES = "http://xml.org/sax/features/external-parameter-entities";
final String ALLOW_EXTERNAL_DTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd";
// Disable DTDs and external entities to protect against XXE
docFactory.setAttribute(DISALLOW_DOCTYPES, true);
docFactory.setAttribute(ALLOW_EXTERNAL_DTD, false);
docFactory.setAttribute(ALLOW_EXTERNAL_GENERAL_ENTITIES, false);
docFactory.setAttribute(ALLOW_EXTERNAL_PARAM_ENTITIES, false);
docFactory.setXIncludeAware(false);
docFactory.setExpandEntityReferences(false);
return docFactory.newDocumentBuilder();
}
/**
* Loads the Notification Services from the given XML configuration file.
*
@ -143,17 +118,14 @@ public class NotificationServiceManager {
*
* @param servicesFile the XML file to load services from.
* @throws IOException if unable to read from the given file
* @throws ParserConfigurationException if unable to parse the given file as XML properly
* @throws SAXException if unable to parse the given file properly
*/
public void loadNotificationServices(final File servicesFile) throws IOException, ParserConfigurationException, SAXException {
final DocumentBuilder docBuilder = createSafeDocumentBuilder();
public void loadNotificationServices(final File servicesFile) throws IOException {
final Map<String, ConfiguredNotificationService> serviceMap = new HashMap<>();
try (final InputStream fis = new FileInputStream(servicesFile);
final InputStream in = new BufferedInputStream(fis)) {
final Document doc = docBuilder.parse(new InputSource(in));
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
final Document doc = documentProvider.parse(in);
final List<Element> serviceElements = getChildElementsByTagName(doc.getDocumentElement(), "service");
logger.debug("Found {} service elements", serviceElements.size());

61
nifi-commons/nifi-security-utils/pom.xml
View File

@ -42,11 +42,6 @@
<artifactId>nifi-security-utils-api</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-kms</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@ -73,62 +68,6 @@
<artifactId>bcrypt</artifactId>
<version>0.9.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
<version>2.7</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes combine.children="append">
<exclude>src/test/resources/xxe_template.xml</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<!-- This profile, activating when compiling on Java versions above 1.8, provides configuration changes to
allow NiFi to be compiled on those JDKs. -->
<id>jigsaw</id>
<activation>
<jdk>(1.8,)</jdk>
</activation>
<dependencies>
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.sun.activation</groupId>
<artifactId>javax.activation</artifactId>
</dependency>
</dependencies>
</profile>
</profiles>
</project>

102
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java
View File

@ -1,102 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.xml;
import java.io.InputStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
public class XmlUtils {
// These features are used to disable processing external entities in the DocumentBuilderFactory to protect against XXE attacks
private static final String DISALLOW_DOCTYPES = "http://apache.org/xml/features/disallow-doctype-decl";
private static final String ALLOW_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities";
private static final String ALLOW_EXTERNAL_PARAM_ENTITIES = "http://xml.org/sax/features/external-parameter-entities";
private static final String ALLOW_EXTERNAL_DTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd";
public static XMLStreamReader createSafeReader(InputStream inputStream) throws XMLStreamException {
if (inputStream == null) {
throw new IllegalArgumentException("The provided input stream cannot be null");
}
return createSafeReader(new StreamSource(inputStream));
}
public static XMLStreamReader createSafeReader(StreamSource source) throws XMLStreamException {
if (source == null) {
throw new IllegalArgumentException("The provided source cannot be null");
}
XMLInputFactory xif = XMLInputFactory.newFactory();
xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
return xif.createXMLStreamReader(source);
}
public static XMLReader createSafeSaxReader(SAXParserFactory saxParserFactory, ContentHandler contentHandler) throws SAXException, ParserConfigurationException {
if (saxParserFactory == null) {
throw new IllegalArgumentException("The provided SAX parser factory cannot be null");
}
if (contentHandler == null) {
throw new IllegalArgumentException("The provided SAX content handler cannot be null");
}
SAXParser saxParser = saxParserFactory.newSAXParser();
XMLReader xmlReader = saxParser.getXMLReader();
xmlReader.setFeature(DISALLOW_DOCTYPES, true);
xmlReader.setFeature(ALLOW_EXTERNAL_GENERAL_ENTITIES, false);
xmlReader.setFeature(ALLOW_EXTERNAL_PARAM_ENTITIES, false);
xmlReader.setContentHandler(contentHandler);
return xmlReader;
}
public static DocumentBuilder createSafeDocumentBuilder(Schema schema, boolean isNamespaceAware) throws ParserConfigurationException {
final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setSchema(schema);
docFactory.setNamespaceAware(isNamespaceAware);
// Disable DTDs and external entities to protect against XXE
docFactory.setAttribute(DISALLOW_DOCTYPES, true);
docFactory.setAttribute(ALLOW_EXTERNAL_DTD, false);
docFactory.setAttribute(ALLOW_EXTERNAL_GENERAL_ENTITIES, false);
docFactory.setAttribute(ALLOW_EXTERNAL_PARAM_ENTITIES, false);
docFactory.setXIncludeAware(false);
docFactory.setExpandEntityReferences(false);
return docFactory.newDocumentBuilder();
}
public static DocumentBuilder createSafeDocumentBuilder(Schema schema) throws ParserConfigurationException {
return createSafeDocumentBuilder(schema, true);
}
public static DocumentBuilder createSafeDocumentBuilder(boolean isNamespaceAware) throws ParserConfigurationException {
return createSafeDocumentBuilder(null, isNamespaceAware);
}
}

100
nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy
View File

@ -1,100 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.xml
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Test
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.xml.sax.SAXParseException
import javax.xml.bind.JAXBContext
import javax.xml.bind.UnmarshalException
import javax.xml.bind.Unmarshaller
import javax.xml.bind.annotation.XmlAccessType
import javax.xml.bind.annotation.XmlAccessorType
import javax.xml.bind.annotation.XmlAttribute
import javax.xml.bind.annotation.XmlRootElement
import javax.xml.parsers.DocumentBuilder
import javax.xml.stream.XMLStreamReader
import static groovy.test.GroovyAssert.shouldFail
class XmlUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(XmlUtilsTest.class)
@BeforeAll
static void setUpOnce() throws Exception {
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Test
void testShouldHandleXXEInUnmarshal() {
// Arrange
final String XXE_TEMPLATE_FILEPATH = "src/test/resources/local_xxe_file.xml"
InputStream templateStream = new File(XXE_TEMPLATE_FILEPATH).newInputStream()
JAXBContext context = JAXBContext.newInstance(XmlObject.class)
// Act
def msg = shouldFail(UnmarshalException) {
Unmarshaller unmarshaller = context.createUnmarshaller()
XMLStreamReader xsr = XmlUtils.createSafeReader(templateStream)
def parsed = unmarshaller.unmarshal(xsr, XmlObject.class)
logger.info("Unmarshalled ${parsed.toString()}")
}
// Assert
logger.expected(msg)
assert msg =~ "XMLStreamException: ParseError "
}
@Test
void testShouldHandleXXEInDocumentBuilder() {
// Arrange
final String XXE_TEMPLATE_FILEPATH = "src/test/resources/local_xxe_file.xml"
DocumentBuilder documentBuilder = XmlUtils.createSafeDocumentBuilder(null)
// Act
def msg = shouldFail(SAXParseException) {
def parsedFlow = documentBuilder.parse(new File(XXE_TEMPLATE_FILEPATH))
logger.info("Parsed ${parsedFlow.toString()}")
}
// Assert
logger.expected(msg)
assert msg =~ "SAXParseException.*DOCTYPE"
}
}
@XmlAccessorType(XmlAccessType.NONE)
@XmlRootElement(name = "object")
class XmlObject {
@XmlAttribute
String name
@XmlAttribute
String description
@XmlAttribute
String groupId
@XmlAttribute
String timestamp
}

5
nifi-commons/nifi-single-user-utils/pom.xml
View File

@ -22,6 +22,11 @@
</parent>
<artifactId>nifi-single-user-utils</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>at.favre.lib</groupId>
<artifactId>bcrypt</artifactId>

12
nifi-commons/nifi-single-user-utils/src/main/java/org/apache/nifi/authentication/single/user/writer/StandardLoginCredentialsWriter.java
View File

@ -17,11 +17,12 @@
package org.apache.nifi.authentication.single.user.writer;
import org.apache.nifi.authentication.single.user.SingleUserCredentials;
import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider;
import javax.xml.stream.XMLEventFactory;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLEventWriter;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.Attribute;
@ -29,6 +30,7 @@ import javax.xml.stream.events.Characters;
import javax.xml.stream.events.EndElement;
import javax.xml.stream.events.StartElement;
import javax.xml.stream.events.XMLEvent;
import javax.xml.transform.stream.StreamSource;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
@ -199,10 +201,8 @@ public class StandardLoginCredentialsWriter implements LoginCredentialsWriter {
return outputFactory.createXMLEventWriter(outputStream);
}
private XMLEventReader getProvidersReader(final InputStream inputStream) throws XMLStreamException {
final XMLInputFactory inputFactory = XMLInputFactory.newFactory();
inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
return inputFactory.createXMLEventReader(inputStream);
private XMLEventReader getProvidersReader(final InputStream inputStream) {
final XMLEventReaderProvider readerProvider = new StandardXMLEventReaderProvider();
return readerProvider.getEventReader(new StreamSource(inputStream));
}
}

5
nifi-commons/nifi-site-to-site-client/pom.xml
View File

@ -51,6 +51,11 @@
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>

12
nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java
View File

@ -25,9 +25,12 @@ import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import org.apache.nifi.security.xml.XmlUtils;
import javax.xml.transform.stream.StreamSource;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
@XmlRootElement
public class ClusterNodeInformation {
@ -65,9 +68,10 @@ public class ClusterNodeInformation {
public static ClusterNodeInformation unmarshal(final InputStream is) throws JAXBException {
try {
final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
final XMLStreamReader xsr = XmlUtils.createSafeReader(is);
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(is));
return (ClusterNodeInformation) unmarshaller.unmarshal(xsr);
} catch (XMLStreamException e) {
} catch (final ProcessingException e) {
throw new JAXBException("Error unmarshalling the cluster node information", e);
}
}

55
nifi-commons/nifi-xml-processing/pom.xml Normal file
View File

@ -0,0 +1,55 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>1.17.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
<packaging>jar</packaging>
<build>
<plugins>
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>4.6.0.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<effort>Max</effort>
<threshold>low</threshold>
<failOnError>true</failOnError>
<plugins>
<plugin>
<groupId>com.h3xstream.findsecbugs</groupId>
<artifactId>findsecbugs-plugin</artifactId>
<version>1.12.0</version>
</plugin>
</plugins>
</configuration>
</plugin>
</plugins>
</build>
</project>

32
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingException.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing;
/**
* General Exception for XML processing problems
*/
public class ProcessingException extends RuntimeException {
/**
* Processing Exception with message and cause for tracing
*
* @param message Error Message
* @param cause Throwable cause for tracing
*/
public ProcessingException(final String message, final Throwable cause) {
super(message, cause);
}
}

53
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingFeature.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing;
import javax.xml.XMLConstants;
/**
* XML Processing Features
*/
public enum ProcessingFeature {
/** Secure Processing */
SECURE_PROCESSING(XMLConstants.FEATURE_SECURE_PROCESSING, true),
/** SAX Namespaces */
SAX_NAMESPACES("http://xml.org/sax/features/namespaces", true),
/** SAX Namespace Prefixes */
SAX_NAMESPACE_PREFIXES("http://xml.org/sax/features/namespace-prefixes", true),
/** Disallow Document Type Declaration */
DISALLOW_DOCTYPE_DECL("http://apache.org/xml/features/disallow-doctype-decl", true);
private final String feature;
private final boolean enabled;
ProcessingFeature(final String feature, final boolean enabled) {
this.feature = feature;
this.enabled = enabled;
}
public String getFeature() {
return feature;
}
public boolean isEnabled() {
return enabled;
}
}

41
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/DocumentProvider.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.parsers;
import org.w3c.dom.Document;
import java.io.InputStream;
/**
* Provider for instances of Document Object Model
*/
public interface DocumentProvider {
/**
* Create new Document
*
* @return Document
*/
Document newDocument();
/**
* Parse InputStream to Document
*
* @param inputStream InputStream to be parsed
* @return Document parsed from stream
*/
Document parse(InputStream inputStream);
}

126
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProvider.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.parsers;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.ProcessingFeature;
import org.w3c.dom.Document;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.validation.Schema;
import java.io.IOException;
import java.io.InputStream;
import java.util.Objects;
/**
* Standard implementation of Document Provider with secure processing enabled
*/
public class StandardDocumentProvider implements DocumentProvider {
private boolean namespaceAware;
private Schema schema;
private ErrorHandler errorHandler;
/**
* Set Error Handler
*
* @param errorHandler Error Handler
*/
public void setErrorHandler(final ErrorHandler errorHandler) {
this.errorHandler = errorHandler;
}
/**
* Set Namespace Aware status on DocumentBuilderFactory
*
* @param namespaceAware Namespace Awareness
*/
public void setNamespaceAware(final boolean namespaceAware) {
this.namespaceAware = namespaceAware;
}
/**
* Set Namespace Aware status on DocumentBuilderFactory
*
* @param schema Schema for validation or null to disable validation
*/
public void setSchema(final Schema schema) {
this.schema = schema;
}
@Override
public Document newDocument() {
final DocumentBuilderFactory documentBuilderFactory = getDocumentBuilderFactory();
try {
documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled());
documentBuilderFactory.setFeature(ProcessingFeature.DISALLOW_DOCTYPE_DECL.getFeature(), ProcessingFeature.DISALLOW_DOCTYPE_DECL.isEnabled());
final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
documentBuilder.setErrorHandler(errorHandler);
return documentBuilder.newDocument();
} catch (final ParserConfigurationException e) {
throw new ProcessingException("Configuration failed", e);
}
}
/**
* Build and return DocumentBuilder
*
* @return DocumentBuilder configured using provided properties
*/
@Override
public Document parse(final InputStream inputStream) {
Objects.requireNonNull(inputStream, "InputStream required");
final DocumentBuilderFactory documentBuilderFactory = getDocumentBuilderFactory();
try {
documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled());
documentBuilderFactory.setFeature(ProcessingFeature.DISALLOW_DOCTYPE_DECL.getFeature(), isDisallowDocumentTypeDeclaration());
final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
documentBuilder.setErrorHandler(errorHandler);
return documentBuilder.parse(inputStream);
} catch (final ParserConfigurationException|SAXException|IOException e) {
throw new ProcessingException("Parsing failed", e);
}
}
protected boolean isDisallowDocumentTypeDeclaration() {
return ProcessingFeature.DISALLOW_DOCTYPE_DECL.isEnabled();
}
private DocumentBuilderFactory getDocumentBuilderFactory() {
final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
documentBuilderFactory.setSchema(schema);
documentBuilderFactory.setNamespaceAware(namespaceAware);
documentBuilderFactory.setXIncludeAware(false);
documentBuilderFactory.setExpandEntityReferences(false);
return documentBuilderFactory;
}
}

33
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/InputSourceParser.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.sax;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
/**
* SAX Input Source Parser
*/
public interface InputSourceParser {
/**
* Parse Input Source using Content Handler
*
* @param inputSource Input Source to be parsed
* @param contentHandler Content Handler used during parsing
*/
void parse(InputSource inputSource, ContentHandler contentHandler);
}

90
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParser.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.sax;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.ProcessingFeature;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import javax.xml.XMLConstants;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import java.io.IOException;
import java.util.Objects;
/**
* Standard implementation of Input Source Parser with secure processing enabled
*/
public class StandardInputSourceParser implements InputSourceParser {
private boolean namespaceAware;
/**
* Set Namespace Aware status on SAXParserFactory
*
* @param namespaceAware Namespace Aware status
*/
public void setNamespaceAware(final boolean namespaceAware) {
this.namespaceAware = namespaceAware;
}
/**
* Parse Input Source using Content Handler
*
* @param inputSource Input Source to be parsed
* @param contentHandler Content Handler used during parsing
*/
@Override
public void parse(final InputSource inputSource, final ContentHandler contentHandler) {
Objects.requireNonNull(inputSource, "InputSource required");
Objects.requireNonNull(contentHandler, "ContentHandler required");
try {
parseInputSource(inputSource, contentHandler);
} catch (final ParserConfigurationException|SAXException e) {
throw new ProcessingException("Parsing failed", e);
}
}
private void parseInputSource(final InputSource inputSource, final ContentHandler contentHandler) throws ParserConfigurationException, SAXException {
final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
saxParserFactory.setNamespaceAware(namespaceAware);
saxParserFactory.setXIncludeAware(false);
saxParserFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled());
saxParserFactory.setFeature(ProcessingFeature.DISALLOW_DOCTYPE_DECL.getFeature(), ProcessingFeature.DISALLOW_DOCTYPE_DECL.isEnabled());
if (namespaceAware) {
saxParserFactory.setFeature(ProcessingFeature.SAX_NAMESPACES.getFeature(), ProcessingFeature.SAX_NAMESPACES.isEnabled());
saxParserFactory.setFeature(ProcessingFeature.SAX_NAMESPACE_PREFIXES.getFeature(), ProcessingFeature.SAX_NAMESPACE_PREFIXES.isEnabled());
}
final SAXParser parser = saxParserFactory.newSAXParser();
final XMLReader reader = parser.getXMLReader();
reader.setContentHandler(contentHandler);
try {
reader.parse(inputSource);
} catch (final IOException e) {
throw new ProcessingException("Parsing failed", e);
}
}
}

51
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProvider.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.stream;
import org.apache.nifi.xml.processing.ProcessingException;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.transform.stream.StreamSource;
import java.util.Objects;
/**
* Standard implementation of XMLStreamReader provider with secure processing enabled
*/
public class StandardXMLEventReaderProvider implements XMLEventReaderProvider {
/**
* Get XML Event Reader
*
* @param streamSource Stream Source for Reader
* @return Configured XML Event Reader
*/
@Override
public XMLEventReader getEventReader(final StreamSource streamSource) {
Objects.requireNonNull(streamSource, "StreamSource required");
final XMLInputFactory inputFactory = XMLInputFactory.newFactory();
inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
try {
return inputFactory.createXMLEventReader(streamSource);
} catch (final XMLStreamException e) {
throw new ProcessingException("Reader creation failed", e);
}
}
}

51
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProvider.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.stream;
import org.apache.nifi.xml.processing.ProcessingException;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import java.util.Objects;
/**
* Standard implementation of XMLStreamReader provider with secure processing enabled
*/
public class StandardXMLStreamReaderProvider implements XMLStreamReaderProvider {
/**
* Get XML Stream Reader with external entities disabled
*
* @param streamSource Stream Source for Reader
* @return Configured XML Stream Reader
*/
@Override
public XMLStreamReader getStreamReader(final StreamSource streamSource) {
Objects.requireNonNull(streamSource, "StreamSource required");
final XMLInputFactory inputFactory = XMLInputFactory.newFactory();
inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
try {
return inputFactory.createXMLStreamReader(streamSource);
} catch (final XMLStreamException e) {
throw new ProcessingException("Reader creation failed", e);
}
}
}

33
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLEventReaderProvider.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.stream;
import javax.xml.stream.XMLEventReader;
import javax.xml.transform.stream.StreamSource;
/**
* Provider for instances of XMLEventReader
*/
public interface XMLEventReaderProvider {
/**
* Get XML Event Reader
*
* @param streamSource Stream Source for Reader
* @return Configured XML Event Reader
*/
XMLEventReader getEventReader(StreamSource streamSource);
}

33
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLStreamReaderProvider.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.stream;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
/**
* Provider for instances of XMLStreamReader
*/
public interface XMLStreamReaderProvider {
/**
* Get XML Stream Reader
*
* @param streamSource Stream Source for Reader
* @return Configured XML Stream Reader
*/
XMLStreamReader getStreamReader(StreamSource streamSource);
}

33
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/SchemaValidator.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.validation;
import javax.xml.transform.Source;
import javax.xml.validation.Schema;
/**
* XML Schema Validator
*/
public interface SchemaValidator {
/**
* Validate Source using Schema
*
* @param schema Schema source for Validator
* @param source Source to be validated
*/
void validate(Schema schema, Source source);
}

59
nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidator.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.validation;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.ProcessingFeature;
import org.xml.sax.SAXException;
import javax.xml.XMLConstants;
import javax.xml.transform.Source;
import javax.xml.validation.Schema;
import javax.xml.validation.Validator;
import java.io.IOException;
import java.util.Objects;
/**
* Standard implementation of XML Schema Validator with secure processing enabled
*/
public class StandardSchemaValidator implements SchemaValidator {
/**
* Validate Source using Schema
*
* @param schema Schema source for Validator
* @param source Source to be validated
*/
@Override
public void validate(final Schema schema, final Source source) {
Objects.requireNonNull(schema, "Schema required");
Objects.requireNonNull(source, "Source required");
final Validator validator = schema.newValidator();
try {
validator.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled());
} catch (final SAXException e) {
throw new ProcessingException("Validator configuration failed", e);
}
try {
validator.validate(source);
} catch (final SAXException|IOException e) {
throw new ProcessingException("Validation failed", e);
}
}
}

65
nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/ResourceProvider.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing;
import java.io.InputStream;
public class ResourceProvider {
private static final String STANDARD_DOCUMENT_DOCTYPE_ENTITY = "/standard-document-doctype-entity.xml";
private static final String STANDARD_DOCUMENT_DOCTYPE = "/standard-document-doctype.xml";
private static final String STANDARD_DOCUMENT = "/standard-document.xml";
private static final String STANDARD_NAMESPACE_DOCUMENT = "/standard-namespace-document.xml";
private static final String STANDARD_NAMESPACE_DOCUMENT_DOCTYPE_ENTITY = "/standard-namespace-document-doctype-entity.xml";
private static final String STANDARD_SCHEMA = "/standard-schema.xsd";
public static InputStream getStandardDocument() {
return getResource(STANDARD_DOCUMENT);
}
public static InputStream getStandardDocumentDocTypeEntity() {
return getResource(STANDARD_DOCUMENT_DOCTYPE_ENTITY);
}
public static InputStream getStandardDocumentDocType() {
return getResource(STANDARD_DOCUMENT_DOCTYPE);
}
public static InputStream getStandardNamespaceDocument() {
return getResource(STANDARD_NAMESPACE_DOCUMENT);
}
public static InputStream getStandardNamespaceDocumentDocTypeEntity() {
return getResource(STANDARD_NAMESPACE_DOCUMENT_DOCTYPE_ENTITY);
}
public static InputStream getStandardSchema() {
return getResource(STANDARD_SCHEMA);
}
private static InputStream getResource(final String path) {
final InputStream resource = ResourceProvider.class.getResourceAsStream(path);
if (resource == null) {
throw new IllegalStateException(String.format("Resource [%s] not found", path));
}
return resource;
}
}

108
nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProviderTest.java Normal file
View File

@ -0,0 +1,108 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.parsers;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.ResourceProvider;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.w3c.dom.Document;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXParseException;
import javax.xml.validation.Schema;
import javax.xml.validation.ValidatorHandler;
import java.io.IOException;
import java.io.InputStream;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class StandardDocumentProviderTest {
@Mock
Schema schema;
@Mock
ValidatorHandler validatorHandler;
@Mock
ErrorHandler errorHandler;
@Test
void testNewDocument() {
final StandardDocumentProvider provider = new StandardDocumentProvider();
final Document document = provider.newDocument();
assertNotNull(document);
}
@Test
void testParseStandard() throws IOException {
final StandardDocumentProvider provider = new StandardDocumentProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocument()) {
final Document document = provider.parse(inputStream);
assertNotNull(document);
}
}
@Test
void testParseDocumentTypeDeclarationException() throws IOException {
final StandardDocumentProvider provider = new StandardDocumentProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) {
assertParsingException(inputStream, provider);
}
}
@Test
void testParseExternalEntityException() throws IOException {
final StandardDocumentProvider provider = new StandardDocumentProvider();
assertParsingException(provider);
}
@Test
void testParseNamespaceAwareSchemaConfiguredExternalEntityException() throws IOException {
when(schema.newValidatorHandler()).thenReturn(validatorHandler);
final StandardDocumentProvider provider = new StandardDocumentProvider();
provider.setNamespaceAware(true);
provider.setSchema(schema);
provider.setErrorHandler(errorHandler);
assertParsingException(provider);
}
private void assertParsingException(final StandardDocumentProvider provider) throws IOException {
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) {
assertParsingException(inputStream, provider);
}
}
private void assertParsingException(final InputStream inputStream, final StandardDocumentProvider provider) {
final ProcessingException processingException = assertThrows(ProcessingException.class, () -> provider.parse(inputStream));
assertInstanceOf(SAXParseException.class, processingException.getCause());
}
}

85
nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParserTest.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.sax;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.ResourceProvider;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXParseException;
import org.xml.sax.helpers.DefaultHandler;
import java.io.IOException;
import java.io.InputStream;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
@ExtendWith(MockitoExtension.class)
public class StandardInputSourceParserTest {
@Mock
ContentHandler contentHandler;
@Test
void testParseStandard() throws IOException {
final StandardInputSourceParser parser = new StandardInputSourceParser();
try (final InputStream inputStream = ResourceProvider.getStandardDocument()) {
parser.parse(new InputSource(inputStream), contentHandler);
}
}
@Test
void testParseDocumentTypeDeclarationException() throws IOException {
final StandardInputSourceParser parser = new StandardInputSourceParser();
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) {
assertParsingException(inputStream, parser);
}
}
@Test
void testParseExternalEntityException() throws IOException {
final StandardInputSourceParser parser = new StandardInputSourceParser();
assertParsingException(parser);
}
@Test
void testParseNamespaceAwareExternalEntityException() throws IOException {
final StandardInputSourceParser parser = new StandardInputSourceParser();
parser.setNamespaceAware(true);
assertParsingException(parser);
}
private void assertParsingException(final StandardInputSourceParser parser) throws IOException {
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) {
assertParsingException(inputStream, parser);
}
}
private void assertParsingException(final InputStream inputStream, final StandardInputSourceParser parser) {
final ProcessingException processingException = assertThrows(ProcessingException.class, () -> parser.parse(new InputSource(inputStream), new DefaultHandler()));
assertInstanceOf(SAXParseException.class, processingException.getCause());
}
}

67
nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProviderTest.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.stream;
import org.apache.nifi.xml.processing.ResourceProvider;
import org.junit.jupiter.api.Test;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLStreamException;
import javax.xml.transform.stream.StreamSource;
import java.io.IOException;
import java.io.InputStream;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class StandardXMLEventReaderProviderTest {
@Test
void testGetEventReaderStandard() throws IOException, XMLStreamException {
final StandardXMLEventReaderProvider provider = new StandardXMLEventReaderProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocument()) {
final XMLEventReader reader = provider.getEventReader(new StreamSource(inputStream));
processReader(reader);
}
}
@Test
void testGetEventReaderStandardDocumentTypeDeclaration() throws IOException {
final StandardXMLEventReaderProvider provider = new StandardXMLEventReaderProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) {
final XMLEventReader reader = provider.getEventReader(new StreamSource(inputStream));
assertDoesNotThrow(() -> processReader(reader));
}
}
@Test
void testGetEventReaderStandardExternalEntityException() throws IOException {
final StandardXMLEventReaderProvider provider = new StandardXMLEventReaderProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) {
final XMLEventReader reader = provider.getEventReader(new StreamSource(inputStream));
assertThrows(XMLStreamException.class, () -> processReader(reader));
}
}
private void processReader(final XMLEventReader reader) throws XMLStreamException {
while (reader.hasNext()) {
reader.nextEvent();
}
}
}

67
nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProviderTest.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.stream;
import org.apache.nifi.xml.processing.ResourceProvider;
import org.junit.jupiter.api.Test;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import java.io.IOException;
import java.io.InputStream;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class StandardXMLStreamReaderProviderTest {
@Test
void testGetStreamReaderStandard() throws IOException, XMLStreamException {
final StandardXMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocument()) {
final XMLStreamReader reader = provider.getStreamReader(new StreamSource(inputStream));
processReader(reader);
}
}
@Test
void testGetStreamReaderStandardDocumentTypeDeclaration() throws IOException {
final StandardXMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) {
final XMLStreamReader reader = provider.getStreamReader(new StreamSource(inputStream));
assertDoesNotThrow(() -> processReader(reader));
}
}
@Test
void testGetStreamReaderStandardExternalEntityException() throws IOException {
final StandardXMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) {
final XMLStreamReader reader = provider.getStreamReader(new StreamSource(inputStream));
assertThrows(XMLStreamException.class, () -> processReader(reader));
}
}
private void processReader(final XMLStreamReader reader) throws XMLStreamException {
while (reader.hasNext()) {
reader.next();
}
}
}

68
nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidatorTest.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.xml.processing.validation;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.ResourceProvider;
import org.junit.jupiter.api.Test;
import org.xml.sax.SAXException;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import java.io.IOException;
import java.io.InputStream;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class StandardSchemaValidatorTest {
private static final String SCHEMA_LANGUAGE = "http://www.w3.org/2001/XMLSchema";
@Test
void testValidate() throws SAXException, IOException {
final SchemaFactory schemaFactory = SchemaFactory.newInstance(SCHEMA_LANGUAGE);
final Schema schema;
try (final InputStream inputStream = ResourceProvider.getStandardSchema()) {
schema = schemaFactory.newSchema(new StreamSource(inputStream));
}
final StandardSchemaValidator validator = new StandardSchemaValidator();
try (final InputStream inputStream = ResourceProvider.getStandardNamespaceDocument()) {
validator.validate(schema, new StreamSource(inputStream));
}
}
@Test
void testValidateExternalEntityException() throws SAXException, IOException {
final SchemaFactory schemaFactory = SchemaFactory.newInstance(SCHEMA_LANGUAGE);
final Schema schema;
try (final InputStream inputStream = ResourceProvider.getStandardSchema()) {
schema = schemaFactory.newSchema(new StreamSource(inputStream));
}
final StandardSchemaValidator validator = new StandardSchemaValidator();
try (final InputStream inputStream = ResourceProvider.getStandardNamespaceDocumentDocTypeEntity()) {
final ProcessingException exception = assertThrows(ProcessingException.class, () -> validator.validate(schema, new StreamSource(inputStream)));
final Throwable cause = exception.getCause();
assertInstanceOf(SAXException.class, cause);
}
}
}

16
nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype-entity.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><!DOCTYPE standard [<!ENTITY entity SYSTEM 'file:///file-not-found'> %entity;]>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<standard>&entity;</standard>

16
nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><!DOCTYPE standard>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<standard />

16
nifi-commons/nifi-xml-processing/src/test/resources/standard-document.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<standard />

16
nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document-doctype-entity.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><!DOCTYPE standard [<!ENTITY entity SYSTEM 'file:///file-not-found'> %entity;]>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<standard xmlns="urn:standard">&entity;</standard>

16
nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<standard xmlns="urn:standard" />

18
nifi-commons/nifi-xml-processing/src/test/resources/standard-schema.xsd Normal file
View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" targetNamespace="urn:standard">
<xs:element name="standard" />
</xs:schema>

1
nifi-commons/pom.xml
View File

@ -65,5 +65,6 @@
<module>nifi-vault-utils</module>
<module>nifi-web-utils</module>
<module>nifi-write-ahead-log</module>
<module>nifi-xml-processing</module>
</modules>
</project>

19
nifi-framework-api/src/main/java/org/apache/nifi/authorization/AbstractPolicyBasedAuthorizer.java
View File

@ -28,6 +28,7 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
@ -36,6 +37,7 @@ import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@ -410,7 +412,7 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer
final PoliciesUsersAndGroups policiesUsersAndGroups = parsePoliciesUsersAndGroups(fingerprint);
if (isInheritable(policiesUsersAndGroups)) {
logger.debug("Inheriting Polciies, Users & Groups");
logger.debug("Inheriting Policies, Users & Groups");
inheritPoliciesUsersAndGroups(policiesUsersAndGroups);
} else {
logger.info("Cannot directly inherit Policies, Users & Groups. Will backup existing Policies, Users & Groups, and then replace with proposed configuration");
@ -427,8 +429,7 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer
final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8);
try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) {
final DocumentBuilder docBuilder = createSafeDocumentBuilder();
final Document document = docBuilder.parse(in);
final Document document = parseFingerprint(in);
final Element rootElement = document.getDocumentElement();
// parse all the users and add them to the current authorizer
@ -451,14 +452,14 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer
Node policyNode = policyNodes.item(i);
accessPolicies.add(parsePolicy((Element) policyNode));
}
} catch (SAXException | ParserConfigurationException | IOException e) {
} catch (final IOException e) {
throw new AuthorizationAccessException("Unable to parse fingerprint", e);
}
return new PoliciesUsersAndGroups(accessPolicies, users, groups);
}
public static DocumentBuilder createSafeDocumentBuilder() throws ParserConfigurationException {
private Document parseFingerprint(final InputStream inputStream) throws IOException {
final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setSchema(null);
docFactory.setNamespaceAware(true);
@ -471,7 +472,13 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer
docFactory.setXIncludeAware(false);
docFactory.setExpandEntityReferences(false);
return docFactory.newDocumentBuilder();
try {
docFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
final DocumentBuilder documentBuilder = docFactory.newDocumentBuilder();
return documentBuilder.parse(inputStream);
} catch (final ParserConfigurationException|SAXException e) {
throw new IOException("Fingerprint parsing failed", e);
}
}
private User parseUser(final Element element) {

6
nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/pom.xml
View File

@ -58,6 +58,12 @@
<version>1.17.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>

26
nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java
View File

@ -28,10 +28,10 @@ import org.apache.nifi.processors.evtx.parser.FileHeaderFactory;
import org.apache.nifi.processors.evtx.parser.MalformedChunkException;
import org.apache.nifi.processors.evtx.parser.Record;
import org.apache.nifi.processors.evtx.parser.bxml.RootNode;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -43,11 +43,8 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLStreamException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -59,8 +56,8 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
@ -159,7 +156,7 @@ public class ParseEvtxTest {
}
@Test
public void testProcessFileGranularity() throws IOException, MalformedChunkException, XMLStreamException {
public void testProcessFileGranularity() throws IOException, MalformedChunkException {
String basename = "basename";
int chunkNum = 5;
int offset = 10001;
@ -203,7 +200,7 @@ public class ParseEvtxTest {
}
@Test
public void testProcessChunkGranularity() throws IOException, MalformedChunkException, XMLStreamException {
public void testProcessChunkGranularity() throws IOException, MalformedChunkException {
String basename = "basename";
int chunkNum = 5;
int offset = 10001;
@ -266,7 +263,7 @@ public class ParseEvtxTest {
}
@Test
public void testProcess1RecordGranularity() throws IOException, MalformedChunkException, XMLStreamException {
public void testProcess1RecordGranularity() throws IOException, MalformedChunkException {
String basename = "basename";
int chunkNum = 5;
int offset = 10001;
@ -340,7 +337,7 @@ public class ParseEvtxTest {
}
@Test
public void fileGranularityLifecycleTest() throws IOException, ParserConfigurationException, SAXException {
public void fileGranularityLifecycleTest() throws IOException {
String baseName = "testFileName";
String name = baseName + ".evtx";
TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class);
@ -374,7 +371,7 @@ public class ParseEvtxTest {
}
@Test
public void chunkGranularityLifecycleTest() throws IOException, ParserConfigurationException, SAXException {
public void chunkGranularityLifecycleTest() throws IOException {
String baseName = "testFileName";
String name = baseName + ".evtx";
TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class);
@ -406,7 +403,7 @@ public class ParseEvtxTest {
}
@Test
public void recordGranularityLifecycleTest() throws IOException, ParserConfigurationException, SAXException {
public void recordGranularityLifecycleTest() throws IOException {
String baseName = "testFileName";
String name = baseName + ".evtx";
TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class);
@ -471,12 +468,13 @@ public class ParseEvtxTest {
testRunner.assertTransferCount(ParseEvtx.REL_SUCCESS, expectedCount);
}
private int validateFlowFiles(List<MockFlowFile> successFlowFiles) throws SAXException, IOException, ParserConfigurationException {
private int validateFlowFiles(List<MockFlowFile> successFlowFiles) {
assertTrue(successFlowFiles.size() > 0);
int totalSize = 0;
for (MockFlowFile successFlowFile : successFlowFiles) {
// Verify valid XML output
Document document = XmlUtils.createSafeDocumentBuilder(false).parse(new ByteArrayInputStream(successFlowFile.toByteArray()));
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
Document document = documentProvider.parse(successFlowFile.getContentStream());
Element documentElement = document.getDocumentElement();
assertEquals(XmlRootNodeHandler.EVENTS, documentElement.getTagName());
NodeList eventNodes = documentElement.getChildNodes();

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java
View File

@ -26,9 +26,11 @@ import org.apache.nifi.authorization.generated.Property;
import org.apache.nifi.bundle.Bundle;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.properties.SensitivePropertyProviderFactoryAware;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.util.file.classloader.ClassLoaderUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
@ -40,7 +42,6 @@ import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
@ -211,12 +212,13 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware
final Schema schema = schemaFactory.newSchema(Authorizers.class.getResource(AUTHORIZERS_XSD));
// attempt to unmarshal
final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(authorizersConfigurationFile));
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(authorizersConfigurationFile));
final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
unmarshaller.setSchema(schema);
final JAXBElement<Authorizers> element = unmarshaller.unmarshal(xsr, Authorizers.class);
return element.getValue();
} catch (XMLStreamException | SAXException | JAXBException e) {
} catch (final ProcessingException | SAXException | JAXBException e) {
throw new Exception("Unable to load the authorizer configuration file at: " + authorizersConfigurationFile.getAbsolutePath(), e);
}
} else {

5
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/pom.xml
View File

@ -47,6 +47,11 @@
<artifactId>nifi-utils</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>

28
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/XmlValidator.java
View File

@ -16,13 +16,13 @@
*/
package org.apache.nifi.documentation.html;
import java.io.IOException;
import java.io.StringReader;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.DocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.junit.Assert;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
/**
* A helper class to validate xml documents.
@ -30,21 +30,21 @@ import org.xml.sax.SAXException;
*
*/
public class XmlValidator {
private static final String DOCTYPE = "<!DOCTYPE html>";
private static final String EMPTY = "";
/**
* Asserts a failure if the provided XML is not valid. <strong>This method does
* not use the "safe" {@link DocumentBuilderFactory} from
* {@code XmlUtils#createSafeDocumentBuilder(Schema, boolean)} because it checks
* generated documentation which contains a doctype. </strong>
* Asserts a failure if the provided XHTML is not valid
*
* @param xml the XML to validate
*/
public static void assertXmlValid(String xml) {
final String html = xml.replace(DOCTYPE, EMPTY);
try {
final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
dbf.setNamespaceAware(true);
dbf.newDocumentBuilder().parse(new InputSource(new StringReader(xml)));
} catch (SAXException | IOException | ParserConfigurationException e) {
final DocumentProvider provider = new StandardDocumentProvider();
provider.parse(new ByteArrayInputStream(html.getBytes(StandardCharsets.UTF_8)));
} catch (final ProcessingException e) {
Assert.fail(e.getMessage());
}
}

5
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/pom.xml
View File

@ -204,6 +204,11 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>

23
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java
View File

@ -30,13 +30,16 @@ import org.apache.nifi.authorization.resource.ResourceType;
import org.apache.nifi.authorization.util.IdentityMapping;
import org.apache.nifi.authorization.util.IdentityMappingUtil;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.user.generated.Users;
import org.apache.nifi.util.FlowInfo;
import org.apache.nifi.util.FlowParser;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.web.api.dto.PortDTO;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@ -51,8 +54,6 @@ import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
@ -501,8 +502,8 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8);
try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) {
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document document = docBuilder.parse(in);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
final Document document = documentProvider.parse(in);
final Element rootElement = document.getDocumentElement();
// parse all the policies and add them to the current access policy provider
@ -511,7 +512,7 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
Node policyNode = policyNodes.item(i);
policies.add(parsePolicy((Element) policyNode));
}
} catch (SAXException | ParserConfigurationException | IOException e) {
} catch (final ProcessingException | IOException e) {
throw new AuthorizationAccessException("Unable to parse fingerprint", e);
}
@ -631,13 +632,14 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
private Authorizations unmarshallAuthorizations() throws JAXBException {
try {
final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(authorizationsFile));
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(authorizationsFile));
final Unmarshaller unmarshaller = JAXB_AUTHORIZATIONS_CONTEXT.createUnmarshaller();
unmarshaller.setSchema(authorizationsSchema);
final JAXBElement<Authorizations> element = unmarshaller.unmarshal(xsr, Authorizations.class);
return element.getValue();
} catch (XMLStreamException e) {
} catch (final ProcessingException e) {
logger.error("Encountered an error reading authorizations file: ", e);
throw new JAXBException("Error reading authorizations file", e);
}
@ -757,8 +759,9 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
final XMLStreamReader xsr;
try {
xsr = XmlUtils.createSafeReader(new StreamSource(authorizedUsersFile));
} catch (XMLStreamException e) {
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile));
} catch (final ProcessingException e) {
logger.error("Encountered an error reading authorized users file: ", e);
throw new JAXBException("Error reading authorized users file", e);
}

26
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java
View File

@ -28,16 +28,18 @@ import org.apache.nifi.authorization.file.tenants.generated.Users;
import org.apache.nifi.authorization.util.IdentityMapping;
import org.apache.nifi.authorization.util.IdentityMappingUtil;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.xml.XMLConstants;
import javax.xml.bind.JAXBContext;
@ -45,8 +47,6 @@ import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
@ -598,8 +598,8 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8);
try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) {
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document document = docBuilder.parse(in);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
final Document document = documentProvider.parse(in);
final Element rootElement = document.getDocumentElement();
// parse all the users and add them to the current user group provider
@ -615,7 +615,7 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
Node groupNode = groupNodes.item(i);
groups.add(parseGroup((Element) groupNode));
}
} catch (SAXException | ParserConfigurationException | IOException e) {
} catch (final ProcessingException | IOException e) {
throw new AuthorizationAccessException("Unable to parse fingerprint", e);
}
@ -724,11 +724,12 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
final Unmarshaller unmarshaller = JAXB_TENANTS_CONTEXT.createUnmarshaller();
unmarshaller.setSchema(tenantsSchema);
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
try {
final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(tenantsFile));
final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(tenantsFile));
final JAXBElement<Tenants> element = unmarshaller.unmarshal(xsr, Tenants.class);
return element.getValue();
} catch (XMLStreamException e) {
} catch (final ProcessingException e) {
throw new JAXBException("Error unmarshalling tenants", e);
}
}
@ -752,10 +753,11 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
throw new AuthorizerCreationException("Legacy Authorized Users File '" + legacyAuthorizedUsersFile + "' does not exists");
}
XMLStreamReader xsr;
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr;
try {
xsr = XmlUtils.createSafeReader(new StreamSource(authorizedUsersFile));
} catch (XMLStreamException e) {
xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile));
} catch (final ProcessingException e) {
throw new AuthorizerCreationException("Error converting the legacy authorizers file", e);
}

66
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/groovy/org/apache/nifi/authorization/FlowParserTest.groovy
View File

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.authorization
import org.apache.nifi.util.FlowParser
import org.junit.After
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
@RunWith(JUnit4.class)
class FlowParserTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(FlowParserTest.class)
@BeforeClass
static void setUpOnce() throws Exception {
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
void setUp() throws Exception {
}
@After
void tearDown() throws Exception {
}
@Test
void testShouldHandleXXEInDocumentBuilder() {
// Arrange
final String XXE_TEMPLATE_FILEPATH = "src/test/resources/flow-with-xxe.xml.gz"
FlowParser fp = new FlowParser()
// Act
def parsedFlow = fp.parse(new File(XXE_TEMPLATE_FILEPATH))
logger.info("Parsed ${parsedFlow.toString()}")
// Assert
// The existing logic logs & swallows any exceptions and returns null
assert !parsedFlow
}
}

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-xxe.xml.gz
View File

Binary file not shown.

5
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/pom.xml
View File

@ -42,6 +42,11 @@
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-core</artifactId>

15
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/src/main/java/org/apache/nifi/authorization/StandardManagedAuthorizer.java
View File

@ -22,15 +22,13 @@ import org.apache.nifi.authorization.exception.AuthorizerCreationException;
import org.apache.nifi.authorization.exception.AuthorizerDestructionException;
import org.apache.nifi.authorization.exception.UninheritableAuthorizationsException;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
@ -239,12 +237,13 @@ public class StandardManagedAuthorizer implements ManagedAuthorizer {
}
}
private final FingerprintHolder parseFingerprint(final String fingerprint) throws AuthorizationAccessException {
private FingerprintHolder parseFingerprint(final String fingerprint) throws AuthorizationAccessException {
final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8);
try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) {
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true);
final Document document = docBuilder.parse(in);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
final Document document = documentProvider.parse(in);
final Element rootElement = document.getDocumentElement();
final NodeList accessPolicyProviderList = rootElement.getElementsByTagName(ACCESS_POLICY_PROVIDER_ELEMENT);
@ -260,7 +259,7 @@ public class StandardManagedAuthorizer implements ManagedAuthorizer {
final Node accessPolicyProvider = accessPolicyProviderList.item(0);
final Node userGroupProvider = userGroupProviderList.item(0);
return new FingerprintHolder(accessPolicyProvider.getTextContent(), userGroupProvider.getTextContent());
} catch (SAXException | ParserConfigurationException | IOException e) {
} catch (final ProcessingException | IOException e) {
throw new AuthorizationAccessException("Unable to parse fingerprint", e);
}
}

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/pom.xml
View File

@ -47,6 +47,16 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-core-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<!-- spring dependencies -->
<dependency>

11
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java
View File

@ -17,15 +17,17 @@
package org.apache.nifi.cluster.protocol;
import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
@ -122,9 +124,10 @@ public class HeartbeatPayload {
public static HeartbeatPayload unmarshal(final InputStream is) throws ProtocolException {
try {
final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
final XMLStreamReader xsr = XmlUtils.createSafeReader(is);
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(is));
return (HeartbeatPayload) unmarshaller.unmarshal(xsr);
} catch (final JAXBException | XMLStreamException e) {
} catch (final JAXBException | ProcessingException e) {
throw new ProtocolException(e);
}
}

26
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java
View File

@ -22,21 +22,17 @@ import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector;
import org.apache.nifi.cluster.protocol.jaxb.message.DataFlowAdapter;
import org.apache.nifi.controller.flow.VersionedDataflow;
import org.apache.nifi.controller.serialization.FlowSerializationException;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.helpers.DefaultHandler;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
@ -48,7 +44,6 @@ import java.util.Set;
*/
@XmlJavaTypeAdapter(DataFlowAdapter.class)
public class StandardDataFlow implements Serializable, DataFlow {
private static final URL FLOW_XSD_RESOURCE = StandardDataFlow.class.getClassLoader().getResource("/FlowConfiguration.xsd");
private static final Logger logger = LoggerFactory.getLogger(StandardDataFlow.class);
private static final long serialVersionUID = 1L;
@ -135,21 +130,19 @@ public class StandardDataFlow implements Serializable, DataFlow {
return null;
}
// create document by parsing proposed flow bytes
try {
// create validating document builder
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true);
docBuilder.setErrorHandler(new DefaultHandler() {
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
documentProvider.setErrorHandler(new DefaultHandler() {
@Override
public void error(final SAXParseException e) {
logger.warn("Schema validation error parsing Flow Configuration at line {}, col {}: {}", e.getLineNumber(), e.getColumnNumber(), e.getMessage());
}
});
// parse flow
return docBuilder.parse(new ByteArrayInputStream(flow));
} catch (final SAXException | ParserConfigurationException | IOException ex) {
throw new FlowSerializationException(ex);
return documentProvider.parse(new ByteArrayInputStream(flow));
} catch (final ProcessingException e) {
throw new FlowSerializationException("Flow parsing failed", e);
}
}
@ -163,8 +156,7 @@ public class StandardDataFlow implements Serializable, DataFlow {
objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory()));
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
final VersionedDataflow versionedDataflow = objectMapper.readValue(flow, VersionedDataflow.class);
return versionedDataflow;
return objectMapper.readValue(flow, VersionedDataflow.class);
} catch (final Exception e) {
throw new FlowSerializationException("Could not parse flow as a VersionedDataflow", e);
}

11
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java
View File

@ -19,14 +19,16 @@ package org.apache.nifi.cluster.protocol.jaxb;
import org.apache.nifi.cluster.protocol.ProtocolContext;
import org.apache.nifi.cluster.protocol.ProtocolMessageMarshaller;
import org.apache.nifi.cluster.protocol.ProtocolMessageUnmarshaller;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
@ -137,10 +139,11 @@ public class JaxbProtocolContext<T> implements ProtocolContext {
final Unmarshaller unmarshaller = jaxbCtx.createUnmarshaller();
final byte[] msg = new byte[totalBytesRead];
buffer.get(msg);
final XMLStreamReader xsr = XmlUtils.createSafeReader(new ByteArrayInputStream(msg));
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(new ByteArrayInputStream(msg)));
return (T) unmarshaller.unmarshal(xsr);
} catch (final JAXBException | XMLStreamException e) {
} catch (final JAXBException | ProcessingException e) {
throw new IOException("Failed unmarshalling protocol message due to: " + e, e);
}

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml
View File

@ -81,6 +81,16 @@
<artifactId>nifi-external-resource-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.registry</groupId>

18
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/state/config/StateManagerConfiguration.java
View File

@ -18,21 +18,22 @@
package org.apache.nifi.controller.state.config;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.nifi.components.state.Scope;
import org.apache.nifi.controller.state.ConfigParseException;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.DomUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.DocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;
public class StateManagerConfiguration {
private final Map<String, StateProviderConfiguration> providers;
@ -62,11 +63,10 @@ public class StateManagerConfiguration {
public static StateManagerConfiguration parse(final File configFile) throws IOException, ConfigParseException {
final Document document;
DocumentBuilder builder;
try {
builder = XmlUtils.createSafeDocumentBuilder(false);
document = builder.parse(configFile);
} catch (ParserConfigurationException | SAXException e) {
try (final InputStream inputStream = new FileInputStream(configFile)) {
final DocumentProvider documentProvider = new StandardDocumentProvider();
document = documentProvider.parse(inputStream);
} catch (final ProcessingException e) {
throw new ConfigParseException("Unable to parse file " + configFile + ", as it does not appear to be a valid XML File", e);
}

5
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml
View File

@ -78,6 +78,11 @@
<artifactId>nifi-repository-encryption</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>

12
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java
View File

@ -27,12 +27,15 @@ import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import org.apache.nifi.cluster.protocol.ProtocolException;
import org.apache.nifi.jaxb.BulletinAdapter;
import org.apache.nifi.reporting.Bulletin;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
/**
* The payload of the bulletins.
@ -80,9 +83,10 @@ public class BulletinsPayload {
public static BulletinsPayload unmarshal(final InputStream is) throws ProtocolException {
try {
final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
final XMLStreamReader xsr = XmlUtils.createSafeReader(is);
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(is));
return (BulletinsPayload) unmarshaller.unmarshal(xsr);
} catch (final JAXBException | XMLStreamException e) {
} catch (final JAXBException | ProcessingException e) {
throw new ProtocolException(e);
}
}

17
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java
View File

@ -48,16 +48,15 @@ import org.apache.nifi.registry.flow.FlowRegistryClient;
import org.apache.nifi.registry.flow.VersionControlInformation;
import org.apache.nifi.remote.PublicPort;
import org.apache.nifi.remote.RemoteGroupPort;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.CharacterFilterUtils;
import org.apache.nifi.util.StringUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
@ -93,8 +92,9 @@ public class StandardFlowSerializer implements FlowSerializer<Document> {
public Document transform(final FlowController controller, final ScheduledStateLookup scheduledStateLookup) throws FlowSerializationException {
try {
// create a new, empty document
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true);
final Document doc = docBuilder.newDocument();
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
final Document doc = documentProvider.newDocument();
// populate document with controller state
final Element rootNode = doc.createElement("flowController");
@ -127,7 +127,7 @@ public class StandardFlowSerializer implements FlowSerializer<Document> {
}
return doc;
} catch (final ParserConfigurationException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) {
} catch (final ProcessingException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) {
throw new FlowSerializationException(e);
}
}
@ -685,10 +685,11 @@ public class StandardFlowSerializer implements FlowSerializer<Document> {
try {
final byte[] serialized = TemplateSerializer.serialize(template.getDetails());
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
final Document document;
try (final InputStream in = new ByteArrayInputStream(serialized)) {
document = docBuilder.parse(in);
document = documentProvider.parse(in);
}
final Node templateNode = element.getOwnerDocument().importNode(document.getDocumentElement(), true);

15
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java
View File

@ -24,11 +24,12 @@ import org.apache.nifi.controller.serialization.FlowFromDOMFactory;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.BundleUtils;
import org.apache.nifi.util.DomUtils;
import org.apache.nifi.web.api.dto.BundleDTO;
import org.apache.nifi.web.api.dto.ControllerServiceDTO;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@ -36,8 +37,6 @@ import org.w3c.dom.Element;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -60,9 +59,9 @@ public class ControllerServiceLoader {
final PropertyEncryptor encryptor, final BulletinRepository bulletinRepo, final boolean autoResumeState, final FlowEncodingVersion encodingVersion) throws IOException {
try (final InputStream in = new BufferedInputStream(serializedStream)) {
final DocumentBuilder builder = XmlUtils.createSafeDocumentBuilder(null);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
builder.setErrorHandler(new org.xml.sax.ErrorHandler() {
documentProvider.setErrorHandler(new org.xml.sax.ErrorHandler() {
@Override
public void fatalError(final SAXParseException err) throws SAXException {
@ -92,15 +91,15 @@ public class ControllerServiceLoader {
}
});
final Document document = builder.parse(in);
final Document document = documentProvider.parse(in);
final Element controllerServices = document.getDocumentElement();
final List<Element> serviceElements = DomUtils.getChildElementsByTagName(controllerServices, "controllerService");
final Map<ControllerServiceNode, Element> controllerServiceMap = ControllerServiceLoader.loadControllerServices(serviceElements, controller, parentGroup, encryptor, encodingVersion);
enableControllerServices(controllerServiceMap, controller, encryptor, autoResumeState, encodingVersion);
return new ArrayList<>(controllerServiceMap.keySet());
} catch (SAXException | ParserConfigurationException sxe) {
throw new IOException(sxe);
} catch (final ProcessingException e) {
throw new IOException("Parsing Controller Services failed", e);
}
}

35
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java
View File

@ -26,27 +26,26 @@ import org.apache.nifi.controller.serialization.FlowFromDOMFactory;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.SensitiveValueEncoder;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.BundleUtils;
import org.apache.nifi.util.DomUtils;
import org.apache.nifi.util.LoggingXmlParserErrorHandler;
import org.apache.nifi.web.api.dto.BundleDTO;
import org.apache.nifi.web.api.dto.ControllerServiceDTO;
import org.apache.nifi.web.api.dto.ReportingTaskDTO;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import org.xml.sax.ErrorHandler;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -83,7 +82,7 @@ public class FingerprintFactory {
private static final String ENCRYPTED_VALUE_PREFIX = "enc{";
private static final String ENCRYPTED_VALUE_SUFFIX = "}";
private final PropertyEncryptor encryptor;
private final DocumentBuilder flowConfigDocBuilder;
private final Schema schema;
private final ExtensionManager extensionManager;
private final SensitiveValueEncoder sensitiveValueEncoder;
@ -95,25 +94,11 @@ public class FingerprintFactory {
this.sensitiveValueEncoder = sensitiveValueEncoder;
final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
final Schema schema;
try {
schema = schemaFactory.newSchema(FingerprintFactory.class.getResource(FLOW_CONFIG_XSD));
} catch (final Exception e) {
throw new RuntimeException("Failed to parse schema for file flow configuration.", e);
}
try {
flowConfigDocBuilder = XmlUtils.createSafeDocumentBuilder(schema, true);
flowConfigDocBuilder.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger));
} catch (final Exception e) {
throw new RuntimeException("Failed to create document builder for flow configuration.", e);
}
}
public FingerprintFactory(final PropertyEncryptor encryptor, final DocumentBuilder docBuilder, final ExtensionManager extensionManager, final SensitiveValueEncoder sensitiveValueEncoder) {
this.encryptor = encryptor;
this.flowConfigDocBuilder = docBuilder;
this.extensionManager = extensionManager;
this.sensitiveValueEncoder = sensitiveValueEncoder;
}
/**
@ -183,9 +168,15 @@ public class FingerprintFactory {
}
try {
return flowConfigDocBuilder.parse(new ByteArrayInputStream(flow));
} catch (final SAXException | IOException ex) {
throw new FingerprintException(ex);
final ErrorHandler errorHandler = new LoggingXmlParserErrorHandler("Flow Configuration", logger);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setSchema(schema);
documentProvider.setNamespaceAware(true);
documentProvider.setErrorHandler(errorHandler);
return documentProvider.parse(new ByteArrayInputStream(flow));
} catch (final ProcessingException e) {
throw new FingerprintException("Flow Parsing failed", e);
}
}

12
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardSnippetDeserializer.java
View File

@ -21,11 +21,14 @@ import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import org.apache.nifi.controller.StandardSnippet;
import org.apache.nifi.controller.serialization.FlowSerializationException;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
public class StandardSnippetDeserializer {
@ -33,10 +36,11 @@ public class StandardSnippetDeserializer {
try {
JAXBContext context = JAXBContext.newInstance(StandardSnippet.class);
Unmarshaller unmarshaller = context.createUnmarshaller();
XMLStreamReader xsr = XmlUtils.createSafeReader(inStream);
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
XMLStreamReader xsr = provider.getStreamReader(new StreamSource(inStream));
JAXBElement<StandardSnippet> snippetElement = unmarshaller.unmarshal(xsr, StandardSnippet.class);
return snippetElement.getValue();
} catch (final JAXBException | XMLStreamException e) {
} catch (final JAXBException | ProcessingException e) {
throw new FlowSerializationException(e);
}
}

11
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java
View File

@ -17,14 +17,15 @@
package org.apache.nifi.persistence;
import org.apache.nifi.controller.serialization.FlowSerializationException;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.web.api.dto.TemplateDTO;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import java.io.InputStream;
@ -46,14 +47,16 @@ public class TemplateDeserializer {
}
public static TemplateDTO deserialize(final StreamSource source) {
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
try {
final XMLStreamReader xsr = XmlUtils.createSafeReader(source);
final XMLStreamReader xsr = provider.getStreamReader(source);
final Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
final JAXBElement<TemplateDTO> templateElement = unmarshaller.unmarshal(xsr, TemplateDTO.class);
final TemplateDTO templateDto = templateElement.getValue();
return templateDto;
} catch (final JAXBException | XMLStreamException e) {
} catch (final JAXBException | ProcessingException e) {
throw new FlowSerializationException(e);
}
}

28
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java
View File

@ -22,20 +22,21 @@ import org.apache.nifi.controller.flow.VersionedDataflow;
import org.apache.nifi.controller.serialization.FlowFromDOMFactory;
import org.apache.nifi.flow.VersionedPort;
import org.apache.nifi.flow.VersionedProcessGroup;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.web.api.dto.PortDTO;
import org.apache.nifi.web.api.dto.PositionDTO;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
@ -117,7 +118,7 @@ public class FlowParser {
return parseJson(flowBytes);
} catch (final SAXException | ParserConfigurationException | IOException ex) {
logger.error("Unable to parse flow {} due to {}", new Object[] { flowPath.toAbsolutePath(), ex });
logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), ex);
return null;
}
}
@ -133,11 +134,14 @@ public class FlowParser {
private FlowInfo parseXml(final byte[] flowBytes) throws ParserConfigurationException, IOException, SAXException {
// create validating document builder
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(flowSchema);
docBuilder.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger));
final ErrorHandler errorHandler = new LoggingXmlParserErrorHandler("Flow Configuration", logger);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setSchema(flowSchema);
documentProvider.setNamespaceAware(true);
documentProvider.setErrorHandler(errorHandler);
// parse the flow
final Document document = docBuilder.parse(new ByteArrayInputStream(flowBytes));
final Document document = documentProvider.parse(new ByteArrayInputStream(flowBytes));
// extract the root group id
final Element rootElement = document.getDocumentElement();
@ -236,11 +240,13 @@ public class FlowParser {
}
// create validating document builder
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(flowSchema);
docBuilder.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger));
return docBuilder.parse(new ByteArrayInputStream(flowBytes));
} catch (final SAXException | ParserConfigurationException | IOException ex) {
logger.error("Unable to parse flow {} due to {}", new Object[]{flowPath.toAbsolutePath(), ex});
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger));
documentProvider.setSchema(flowSchema);
documentProvider.setNamespaceAware(true);
return documentProvider.parse(new ByteArrayInputStream(flowBytes));
} catch (final ProcessingException | IOException e) {
logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), e);
return null;
}
}

74
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java
View File

@ -33,21 +33,14 @@ import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.StandardExtensionDiscoveringManager;
import org.apache.nifi.remote.RemoteGroupPort;
import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.parsers.DocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
@ -55,7 +48,6 @@ import java.util.Collections;
import java.util.Optional;
import static org.apache.nifi.controller.serialization.ScheduledStateLookup.IDENTITY_LOOKUP;
import static org.apache.nifi.fingerprint.FingerprintFactory.FLOW_CONFIG_XSD;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
@ -170,12 +162,11 @@ public class FingerprintFactoryTest {
}
@Test
public void testPublicPortWithDifferentFingerprintInAccessPolicies() throws IOException, ParserConfigurationException, SAXException {
public void testPublicPortWithDifferentFingerprintInAccessPolicies() throws IOException {
final String f1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
assertEquals(2, StringUtils.countMatches(f1, "user1group1"));
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document document = docBuilder.parse(new File("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml"));
final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml");
final Element rootProcessGroup = document.getDocumentElement();
final StringBuilder sb = new StringBuilder();
@ -190,9 +181,8 @@ public class FingerprintFactoryTest {
}
@Test
public void testPublicPortWithNoAccessPoliciesFingerprint() throws ParserConfigurationException, IOException, SAXException {
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document document = docBuilder.parse(new File("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml"));
public void testPublicPortWithNoAccessPoliciesFingerprint() throws IOException {
final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml");
final Element rootProcessGroup = document.getDocumentElement();
final StringBuilder sb = new StringBuilder();
@ -203,52 +193,21 @@ public class FingerprintFactoryTest {
assertTrue(fingerprint.contains("NO_GROUP_ACCESS_CONTROL"));
}
@Test
public void testSchemaValidation() throws IOException {
FingerprintFactory fp = new FingerprintFactory(null, getValidatingDocumentBuilder(), extensionManager, null);
fp.createFingerprint(getResourceBytes("/nifi/fingerprint/validating-flow.xml"), null);
private Document getDocument(final String filePath) throws IOException {
try (final FileInputStream inputStream = new FileInputStream(filePath)) {
final DocumentProvider documentProvider = new StandardDocumentProvider();
return documentProvider.parse(inputStream);
}
}
private byte[] getResourceBytes(final String resource) throws IOException {
return IOUtils.toByteArray(FingerprintFactoryTest.class.getResourceAsStream(resource));
}
private DocumentBuilder getValidatingDocumentBuilder() {
final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
final Schema schema;
try {
schema = schemaFactory.newSchema(FingerprintFactory.class.getResource(FLOW_CONFIG_XSD));
} catch (final Exception e) {
throw new RuntimeException("Failed to parse schema for file flow configuration.", e);
}
try {
DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(schema, true);
docBuilder.setErrorHandler(new ErrorHandler() {
@Override
public void warning(SAXParseException e) throws SAXException {
throw e;
}
@Override
public void error(SAXParseException e) throws SAXException {
throw e;
}
@Override
public void fatalError(SAXParseException e) throws SAXException {
throw e;
}
});
return docBuilder;
} catch (final Exception e) {
throw new RuntimeException("Failed to create document builder for flow configuration.", e);
}
}
private <T> Element serializeElement(final PropertyEncryptor encryptor, final Class<T> componentClass, final T component,
final String serializerMethodName, ScheduledStateLookup scheduledStateLookup) throws Exception {
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document doc = docBuilder.newDocument();
final DocumentProvider documentProvider = new StandardDocumentProvider();
final Document doc = documentProvider.newDocument();
final FlowSerializer flowSerializer = new StandardFlowSerializer(encryptor);
final Method serializeMethod = StandardFlowSerializer.class.getDeclaredMethod(serializerMethodName,
@ -394,9 +353,8 @@ public class FingerprintFactoryTest {
}
@Test
public void testControllerServicesIncludedInGroupFingerprint() throws ParserConfigurationException, IOException, SAXException {
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document document = docBuilder.parse(new File("src/test/resources/nifi/fingerprint/group-with-controller-services.xml"));
public void testControllerServicesIncludedInGroupFingerprint() throws IOException {
final Document document = getDocument("src/test/resources/nifi/fingerprint/group-with-controller-services.xml");
final Element processGroup = document.getDocumentElement();
final StringBuilder sb = new StringBuilder();

7
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java
View File

@ -16,11 +16,12 @@
*/
package org.apache.nifi.persistence;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.ComponentIdGenerator;
import org.apache.nifi.web.api.dto.FlowSnippetDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.api.dto.TemplateDTO;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.diff.EditList;
import org.eclipse.jgit.diff.HistogramDiff;
@ -32,6 +33,7 @@ import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@ -69,7 +71,8 @@ public class TemplateSerializerTest {
ByteArrayInputStream in = new ByteArrayInputStream(serTemplate);
JAXBContext context = JAXBContext.newInstance(TemplateDTO.class);
Unmarshaller unmarshaller = context.createUnmarshaller();
XMLStreamReader xsr = XmlUtils.createSafeReader(in);
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
XMLStreamReader xsr = provider.getStreamReader(new StreamSource(in));
JAXBElement<TemplateDTO> templateElement = unmarshaller.unmarshal(xsr, TemplateDTO.class);
TemplateDTO deserTemplate = templateElement.getValue();

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/pom.xml
View File

@ -195,6 +195,10 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-security</artifactId>

18
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java
View File

@ -34,6 +34,8 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.web.api.dto.ProcessorConfigDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.dao.ProcessorDAO;
import org.apache.nifi.xml.processing.parsers.DocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
@ -43,11 +45,9 @@ import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.StringReader;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collection;
@ -180,16 +180,12 @@ public class ProcessorAuditor extends NiFiAuditor {
try {
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(newValue));
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
Document doc = dBuilder.parse(is);
final DocumentProvider documentProvider = new StandardDocumentProvider();
Document doc = documentProvider.parse(new ByteArrayInputStream(newValue.getBytes(StandardCharsets.UTF_8)));
NodeList nList = doc.getChildNodes();
final Map<String, Node> xmlDumpNew = new HashMap<>();
getItemPaths(nList, ""+doc.getNodeName(), xmlDumpNew);
is.setCharacterStream(new StringReader(oldValue));
doc = dBuilder.parse(is);
doc = documentProvider.parse(new ByteArrayInputStream(oldValue.getBytes(StandardCharsets.UTF_8)));
nList = doc.getChildNodes();
final Map<String, Node> xmlDumpOld = new HashMap<>();
getItemPaths(nList, ""+doc.getNodeName(), xmlDumpOld);

8
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java
View File

@ -78,6 +78,8 @@ import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.stream.StreamSource;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.nifi.authorization.AuthorizableLookup;
@ -113,7 +115,6 @@ import org.apache.nifi.flow.VersionedProcessGroup;
import org.apache.nifi.registry.variable.VariableRegistryUpdateRequest;
import org.apache.nifi.registry.variable.VariableRegistryUpdateStep;
import org.apache.nifi.remote.util.SiteToSiteRestApiClient;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.web.ResourceNotFoundException;
import org.apache.nifi.web.Revision;
import org.apache.nifi.web.api.dto.AffectedComponentDTO;
@ -173,6 +174,8 @@ import org.apache.nifi.web.api.request.ClientIdParameter;
import org.apache.nifi.web.api.request.LongParameter;
import org.apache.nifi.web.security.token.NiFiAuthenticationToken;
import org.apache.nifi.web.util.Pause;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import org.glassfish.jersey.media.multipart.FormDataParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -3814,7 +3817,8 @@ public class ProcessGroupResource extends FlowUpdateResource<ProcessGroupImportE
// TODO: Potentially refactor the template parsing to a service layer outside of the resource for web request handling
JAXBContext context = JAXBContext.newInstance(TemplateDTO.class);
Unmarshaller unmarshaller = context.createUnmarshaller();
XMLStreamReader xsr = XmlUtils.createSafeReader(in);
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
XMLStreamReader xsr = provider.getStreamReader(new StreamSource(in));
JAXBElement<TemplateDTO> templateElement = unmarshaller.unmarshal(xsr, TemplateDTO.class);
template = templateElement.getValue();
} catch (JAXBException jaxbe) {

6
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java
View File

@ -33,8 +33,9 @@ import org.apache.nifi.bundle.Bundle;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.NarCloseable;
import org.apache.nifi.properties.SensitivePropertyProviderFactoryAware;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.FactoryBean;
import org.xml.sax.SAXException;
@ -138,7 +139,8 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF
final Schema schema = schemaFactory.newSchema(LoginIdentityProviders.class.getResource(LOGIN_IDENTITY_PROVIDERS_XSD));
// attempt to unmarshal
XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(loginIdentityProvidersConfigurationFile));
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
XMLStreamReader xsr = provider.getStreamReader(new StreamSource(loginIdentityProvidersConfigurationFile));
final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller();
unmarshaller.setSchema(schema);
final JAXBElement<LoginIdentityProviders> element = unmarshaller.unmarshal(xsr, LoginIdentityProviders.class);

5
nifi-nar-bundles/nifi-framework-bundle/pom.xml
View File

@ -60,6 +60,11 @@
<artifactId>nifi-security-utils-api</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-expression-language</artifactId>

2
nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
View File

@ -186,7 +186,7 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>

18
nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/ManagedRangerAuthorizer.java
View File

@ -23,8 +23,6 @@ import java.io.IOException;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
@ -45,12 +43,12 @@ import org.apache.nifi.authorization.exception.AuthorizationAccessException;
import org.apache.nifi.authorization.exception.AuthorizerCreationException;
import org.apache.nifi.authorization.exception.AuthorizerDestructionException;
import org.apache.nifi.authorization.exception.UninheritableAuthorizationsException;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements ManagedAuthorizer {
private static final String USER_GROUP_PROVIDER_ELEMENT = "userGroupProvider";
@ -128,8 +126,8 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man
final StringWriter out = new StringWriter();
try {
// create the document
final DocumentBuilder documentBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document document = documentBuilder.newDocument();
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
final Document document = documentProvider.newDocument();
// create the root element
final Element managedRangerAuthorizationsElement = document.createElement("managedRangerAuthorizations");
@ -146,7 +144,7 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man
final Transformer transformer = TransformerFactory.newInstance().newTransformer();
transformer.transform(new DOMSource(document), new StreamResult(out));
} catch (ParserConfigurationException | TransformerException e) {
} catch (final ProcessingException | TransformerException e) {
throw new AuthorizationAccessException("Unable to generate fingerprint", e);
}
@ -192,8 +190,8 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man
final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8);
try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) {
final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false);
final Document document = docBuilder.parse(in);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
final Document document = documentProvider.parse(in);
final Element rootElement = document.getDocumentElement();
final NodeList userGroupProviderList = rootElement.getElementsByTagName(USER_GROUP_PROVIDER_ELEMENT);
@ -203,7 +201,7 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man
final Node userGroupProvider = userGroupProviderList.item(0);
return userGroupProvider.getTextContent();
} catch (SAXException | ParserConfigurationException | IOException e) {
} catch (final ProcessingException | IOException e) {
throw new AuthorizationAccessException("Unable to parse fingerprint", e);
}
}

5
nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/pom.xml
View File

@ -46,5 +46,10 @@
<artifactId>nifi-single-user-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

10
nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/src/main/java/org/apache/nifi/authorization/single/user/SingleUserAuthorizer.java
View File

@ -26,14 +26,16 @@ import org.apache.nifi.authorization.annotation.AuthorizerContext;
import org.apache.nifi.authorization.exception.AuthorizationAccessException;
import org.apache.nifi.authorization.exception.AuthorizerCreationException;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.StartElement;
import javax.xml.stream.events.XMLEvent;
import javax.xml.transform.stream.StreamSource;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -149,9 +151,7 @@ public class SingleUserAuthorizer implements Authorizer {
}
private XMLEventReader getProvidersReader(final InputStream inputStream) throws XMLStreamException {
final XMLInputFactory inputFactory = XMLInputFactory.newFactory();
inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
return inputFactory.createXMLEventReader(inputStream);
final XMLEventReaderProvider readerProvider = new StandardXMLEventReaderProvider();
return readerProvider.getEventReader(new StreamSource(inputStream));
}
}

5
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
View File

@ -158,6 +158,11 @@
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.hierynomus</groupId>
<artifactId>sshj</artifactId>

161
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java
View File

@ -22,11 +22,9 @@ import static javax.xml.xpath.XPathConstants.STRING;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -46,12 +44,11 @@ import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.TransformerFactoryConfigurationError;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import javax.xml.xpath.XPathFactoryConfigurationException;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.EventDriven;
@ -59,6 +56,9 @@ import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.SystemResource;
import org.apache.nifi.annotation.behavior.SystemResourceConsideration;
import org.apache.nifi.annotation.behavior.SystemResourceConsiderations;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
@ -76,16 +76,14 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.apache.nifi.processors.standard.xml.DocumentTypeAllowedDocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.w3c.dom.Document;
import net.sf.saxon.lib.NamespaceConstant;
import net.sf.saxon.xpath.XPathEvaluator;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLReaderFactory;
import net.sf.saxon.xpath.XPathFactoryImpl;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@EventDriven
@SideEffectFree
@ -105,6 +103,9 @@ import org.xml.sax.helpers.XMLReaderFactory;
@WritesAttribute(attribute = "user-defined", description = "This processor adds user-defined attributes if the <Destination> property is set to flowfile-attribute.")
@DynamicProperty(name = "A FlowFile attribute(if <Destination> is set to 'flowfile-attribute'", value = "An XPath expression", description = "If <Destination>='flowfile-attribute' "
+ "then the FlowFile attribute is set to the result of the XPath Expression. If <Destination>='flowfile-content' then the FlowFile content is set to the result of the XPath Expression.")
@SystemResourceConsiderations({
@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "Processing requires reading the entire FlowFile into memory")
})
public class EvaluateXPath extends AbstractProcessor {
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
@ -137,7 +138,7 @@ public class EvaluateXPath extends AbstractProcessor {
.description("Specifies whether or not the XML content should be validated against the DTD.")
.required(true)
.allowableValues("true", "false")
.defaultValue("true")
.defaultValue("false")
.build();
public static final Relationship REL_MATCH = new Relationship.Builder()
@ -162,10 +163,6 @@ public class EvaluateXPath extends AbstractProcessor {
private final AtomicReference<XPathFactory> factoryRef = new AtomicReference<>();
static {
System.setProperty("javax.xml.xpath.XPathFactory:" + NamespaceConstant.OBJECT_MODEL_SAXON, "net.sf.saxon.xpath.XPathFactoryImpl");
}
@Override
protected void init(final ProcessorInitializationContext context) {
final Set<Relationship> relationships = new HashSet<>();
@ -215,8 +212,8 @@ public class EvaluateXPath extends AbstractProcessor {
}
@OnScheduled
public void initializeXPathFactory() throws XPathFactoryConfigurationException {
factoryRef.set(XPathFactory.newInstance(NamespaceConstant.OBJECT_MODEL_SAXON));
public void initializeXPathFactory() {
factoryRef.set(new XPathFactoryImpl());
}
@Override
@ -231,7 +228,6 @@ public class EvaluateXPath extends AbstractProcessor {
}
@Override
@SuppressWarnings("unchecked")
public void onTrigger(final ProcessContext context, final ProcessSession session) {
final List<FlowFile> flowFiles = session.get(50);
if (flowFiles.isEmpty()) {
@ -239,23 +235,6 @@ public class EvaluateXPath extends AbstractProcessor {
}
final ComponentLog logger = getLogger();
final XMLReader xmlReader;
try {
xmlReader = XMLReaderFactory.createXMLReader();
} catch (SAXException e) {
logger.error("Error while constructing XMLReader {}", new Object[]{e});
throw new ProcessException(e.getMessage());
}
if (!context.getProperty(VALIDATE_DTD).asBoolean()) {
xmlReader.setEntityResolver(new EntityResolver() {
@Override
public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
return new InputSource(new StringReader(""));
}
});
}
final XPathFactory factory = factoryRef.get();
final XPathEvaluator xpathEvaluator = (XPathEvaluator) factory.newXPath();
@ -274,15 +253,6 @@ public class EvaluateXPath extends AbstractProcessor {
}
}
final XPathExpression slashExpression;
try {
slashExpression = xpathEvaluator.compile("/");
} catch (XPathExpressionException e) {
logger.error("unable to compile XPath expression due to {}", new Object[]{e});
session.transfer(flowFiles, REL_FAILURE);
return;
}
final String destination = context.getProperty(DESTINATION).getValue();
final QName returnType;
@ -306,27 +276,25 @@ public class EvaluateXPath extends AbstractProcessor {
throw new IllegalStateException("There are no other return types...");
}
final boolean validatingDeclaration = context.getProperty(VALIDATE_DTD).asBoolean();
flowFileLoop:
for (FlowFile flowFile : flowFiles) {
final AtomicReference<Throwable> error = new AtomicReference<>(null);
final AtomicReference<Source> sourceRef = new AtomicReference<>(null);
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream rawIn) throws IOException {
try {
session.read(flowFile, rawIn -> {
try (final InputStream in = new BufferedInputStream(rawIn)) {
final List<Source> rootList = (List<Source>) slashExpression.evaluate(new SAXSource(xmlReader,
new InputSource(in)), NODESET);
sourceRef.set(rootList.get(0));
} catch (final Exception e) {
error.set(e);
final StandardDocumentProvider documentProvider = validatingDeclaration
? new DocumentTypeAllowedDocumentProvider()
: new StandardDocumentProvider();
final Document document = documentProvider.parse(in);
sourceRef.set(new DOMSource(document));
}
}
});
if (error.get() != null) {
logger.error("unable to evaluate XPath against {} due to {}; routing to 'failure'",
new Object[]{flowFile, error.get()});
});
} catch (final Exception e) {
logger.error("Input parsing failed {}", flowFile, e);
session.transfer(flowFile, REL_FAILURE);
continue;
}
@ -334,69 +302,60 @@ public class EvaluateXPath extends AbstractProcessor {
final Map<String, String> xpathResults = new HashMap<>();
for (final Map.Entry<String, XPathExpression> entry : attributeToXPathMap.entrySet()) {
Object result = null;
Object result;
try {
result = entry.getValue().evaluate(sourceRef.get(), returnType);
if (result == null) {
continue;
}
} catch (final XPathExpressionException e) {
logger.error("failed to evaluate XPath for {} for Property {} due to {}; routing to failure",
new Object[]{flowFile, entry.getKey(), e});
logger.error("XPath Property [{}] evaluation on {} failed", flowFile, entry.getKey(), e);
session.transfer(flowFile, REL_FAILURE);
continue flowFileLoop;
}
if (returnType == NODESET) {
List<Source> nodeList = (List<Source>) result;
if (nodeList.isEmpty()) {
logger.info("Routing {} to 'unmatched'", new Object[]{flowFile});
final NodeList nodeList = (NodeList) result;
if (nodeList.getLength() == 0) {
logger.info("XPath evaluation on {} produced no results", flowFile);
session.transfer(flowFile, REL_NO_MATCH);
continue flowFileLoop;
} else if (nodeList.size() > 1) {
logger.error("Routing {} to 'failure' because the XPath evaluated to {} XML nodes",
new Object[]{flowFile, nodeList.size()});
} else if (nodeList.getLength() > 1) {
logger.error("XPath evaluation on {} produced unexpected results [{}]", flowFile, nodeList.getLength());
session.transfer(flowFile, REL_FAILURE);
continue flowFileLoop;
}
final Source sourceNode = nodeList.get(0);
final Node firstNode = nodeList.item(0);
final Source sourceNode = new DOMSource(firstNode);
if (DESTINATION_ATTRIBUTE.equals(destination)) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
doTransform(sourceNode, baos);
xpathResults.put(entry.getKey(), baos.toString("UTF-8"));
} catch (UnsupportedEncodingException e) {
throw new ProcessException(e);
xpathResults.put(entry.getKey(), new String(baos.toByteArray(), StandardCharsets.UTF_8));
} catch (TransformerException e) {
error.set(e);
}
} else if (DESTINATION_CONTENT.equals(destination)) {
flowFile = session.write(flowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream rawOut) throws IOException {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
doTransform(sourceNode, out);
} catch (TransformerException e) {
error.set(e);
}
flowFile = session.write(flowFile, rawOut -> {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
doTransform(sourceNode, out);
} catch (TransformerException e) {
error.set(e);
}
});
}
} else if (returnType == STRING) {
} else {
final String resultString = (String) result;
if (DESTINATION_ATTRIBUTE.equals(destination)) {
xpathResults.put(entry.getKey(), resultString);
} else if (DESTINATION_CONTENT.equals(destination)) {
flowFile = session.write(flowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream rawOut) throws IOException {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
out.write(resultString.getBytes("UTF-8"));
}
flowFile = session.write(flowFile, rawOut -> {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
out.write(resultString.getBytes(StandardCharsets.UTF_8));
}
});
}
@ -407,18 +366,16 @@ public class EvaluateXPath extends AbstractProcessor {
if (DESTINATION_ATTRIBUTE.equals(destination)) {
flowFile = session.putAllAttributes(flowFile, xpathResults);
final Relationship destRel = xpathResults.isEmpty() ? REL_NO_MATCH : REL_MATCH;
logger.info("Successfully evaluated XPaths against {} and found {} matches; routing to {}",
new Object[]{flowFile, xpathResults.size(), destRel.getName()});
logger.info("XPath evaluation on {} completed with results [{}]: content updated", flowFile, xpathResults.size());
session.transfer(flowFile, destRel);
session.getProvenanceReporter().modifyAttributes(flowFile);
} else if (DESTINATION_CONTENT.equals(destination)) {
logger.info("Successfully updated content for {}; routing to 'matched'", new Object[]{flowFile});
logger.info("XPath evaluation on {} completed: content updated", flowFile);
session.transfer(flowFile, REL_MATCH);
session.getProvenanceReporter().modifyContent(flowFile);
}
} else {
logger.error("Failed to write XPath result for {} due to {}; routing original to 'failure'",
new Object[]{flowFile, error.get()});
logger.error("XPath evaluation on {} failed", flowFile, error.get());
session.transfer(flowFile, REL_FAILURE);
}
}
@ -443,19 +400,19 @@ public class EvaluateXPath extends AbstractProcessor {
final AtomicReference<TransformerException> error = new AtomicReference<>(null);
transformer.setErrorListener(new ErrorListener() {
@Override
public void warning(final TransformerException exception) throws TransformerException {
logger.warn("Encountered warning from XPath Engine: ", new Object[]{exception.toString(), exception});
public void warning(final TransformerException exception) {
logger.warn("Encountered warning from XPath Engine", exception);
}
@Override
public void error(final TransformerException exception) throws TransformerException {
logger.error("Encountered error from XPath Engine: ", new Object[]{exception.toString(), exception});
public void error(final TransformerException exception) {
logger.error("Encountered error from XPath Engine", exception);
error.set(exception);
}
@Override
public void fatalError(final TransformerException exception) throws TransformerException {
logger.error("Encountered warning from XPath Engine: ", new Object[]{exception.toString(), exception});
public void fatalError(final TransformerException exception) {
logger.error("Encountered warning from XPath Engine", exception);
error.set(exception);
}
});
@ -471,7 +428,7 @@ public class EvaluateXPath extends AbstractProcessor {
@Override
public ValidationResult validate(final String subject, final String input, final ValidationContext validationContext) {
try {
XPathFactory factory = XPathFactory.newInstance(NamespaceConstant.OBJECT_MODEL_SAXON);
XPathFactory factory = new XPathFactoryImpl();
final XPathEvaluator evaluator = (XPathEvaluator) factory.newXPath();
String error = null;
@ -484,7 +441,7 @@ public class EvaluateXPath extends AbstractProcessor {
return new ValidationResult.Builder().input(input).subject(subject).valid(error == null).explanation(error).build();
} catch (final Exception e) {
return new ValidationResult.Builder().input(input).subject(subject).valid(false)
.explanation("Unable to initialize XPath engine due to " + e.toString()).build();
.explanation("Unable to initialize XPath engine due to " + e).build();
}
}
}

132
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java
View File

@ -22,7 +22,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
@ -39,9 +38,8 @@ import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.TransformerFactoryConfigurationError;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import net.sf.saxon.s9api.DOMDestination;
import net.sf.saxon.s9api.Processor;
import net.sf.saxon.s9api.SaxonApiException;
import net.sf.saxon.s9api.XQueryCompiler;
@ -56,6 +54,9 @@ import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.SystemResource;
import org.apache.nifi.annotation.behavior.SystemResourceConsideration;
import org.apache.nifi.annotation.behavior.SystemResourceConsiderations;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
@ -72,16 +73,10 @@ import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.processors.standard.xml.DocumentTypeAllowedDocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.w3c.dom.Document;
import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLReaderFactory;
@EventDriven
@SideEffectFree
@ -103,6 +98,9 @@ import org.xml.sax.helpers.XMLReaderFactory;
@WritesAttribute(attribute = "user-defined", description = "This processor adds user-defined attributes if the <Destination> property is set to flowfile-attribute .")
@DynamicProperty(name = "A FlowFile attribute(if <Destination> is set to 'flowfile-attribute'", value = "An XQuery", description = "If <Destination>='flowfile-attribute' "
+ "then the FlowFile attribute is set to the result of the XQuery. If <Destination>='flowfile-content' then the FlowFile content is set to the result of the XQuery.")
@SystemResourceConsiderations({
@SystemResourceConsideration(resource = SystemResource.MEMORY, description = "Processing requires reading the entire FlowFile into memory")
})
public class EvaluateXQuery extends AbstractProcessor {
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
@ -112,8 +110,6 @@ public class EvaluateXQuery extends AbstractProcessor {
public static final String OUTPUT_METHOD_HTML = "html";
public static final String OUTPUT_METHOD_TEXT = "text";
public static final String UTF8 = "UTF-8";
public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
.name("Destination")
.description(
@ -156,7 +152,7 @@ public class EvaluateXQuery extends AbstractProcessor {
.description("Specifies whether or not the XML content should be validated against the DTD.")
.required(true)
.allowableValues("true", "false")
.defaultValue("true")
.defaultValue("false")
.build();
public static final Relationship REL_MATCH = new Relationship.Builder()
@ -248,24 +244,6 @@ public class EvaluateXQuery extends AbstractProcessor {
final Map<String, XQueryExecutable> attributeToXQueryMap = new HashMap<>();
final Processor proc = new Processor(false);
final XMLReader xmlReader;
try {
xmlReader = XMLReaderFactory.createXMLReader();
} catch (SAXException e) {
logger.error("Error while constructing XMLReader {}", new Object[]{e});
throw new ProcessException(e.getMessage());
}
if (!context.getProperty(VALIDATE_DTD).asBoolean()) {
xmlReader.setEntityResolver(new EntityResolver() {
@Override
public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
return new InputSource(new StringReader(""));
}
});
}
final XQueryCompiler comp = proc.newXQueryCompiler();
for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
@ -281,16 +259,8 @@ public class EvaluateXQuery extends AbstractProcessor {
}
}
final XQueryExecutable slashExpression;
try {
slashExpression = comp.compile("/");
} catch (SaxonApiException e) {
logger.error("unable to compile XQuery expression due to {}", new Object[]{e});
session.transfer(flowFileBatch, REL_FAILURE);
return;
}
final String destination = context.getProperty(DESTINATION).getValue();
final boolean validateDeclaration = context.getProperty(VALIDATE_DTD).asBoolean();
flowFileLoop:
for (FlowFile flowFile : flowFileBatch) {
@ -299,28 +269,20 @@ public class EvaluateXQuery extends AbstractProcessor {
return;
}
final AtomicReference<Throwable> error = new AtomicReference<>(null);
final AtomicReference<XdmNode> sourceRef = new AtomicReference<>(null);
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream rawIn) throws IOException {
final AtomicReference<DOMSource> sourceRef = new AtomicReference<>(null);
try {
session.read(flowFile, rawIn -> {
try (final InputStream in = new BufferedInputStream(rawIn)) {
XQueryEvaluator qe = slashExpression.load();
qe.setSource(new SAXSource(xmlReader, new InputSource(in)));
Document dom = XmlUtils.createSafeDocumentBuilder(true).newDocument();
qe.run(new DOMDestination(dom));
XdmNode rootNode = proc.newDocumentBuilder().wrap(dom);
sourceRef.set(rootNode);
} catch (final Exception e) {
error.set(e);
final StandardDocumentProvider documentProvider = validateDeclaration
? new DocumentTypeAllowedDocumentProvider()
: new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
final Document document = documentProvider.parse(in);
sourceRef.set(new DOMSource(document));
}
}
});
if (error.get() != null) {
logger.error("unable to evaluate XQuery against {} due to {}; routing to 'failure'",
new Object[]{flowFile, error.get()});
});
} catch (final Exception e) {
logger.error("Input parsing failed {}", flowFile, e);
session.transfer(flowFile, REL_FAILURE);
continue;
}
@ -331,7 +293,7 @@ public class EvaluateXQuery extends AbstractProcessor {
for (final Map.Entry<String, XQueryExecutable> entry : attributeToXQueryMap.entrySet()) {
try {
XQueryEvaluator qe = entry.getValue().load();
qe.setContextItem(sourceRef.get());
qe.setSource(sourceRef.get());
XdmValue result = qe.evaluate();
if (DESTINATION_ATTRIBUTE.equals(destination)) {
@ -346,33 +308,27 @@ public class EvaluateXQuery extends AbstractProcessor {
}
} else { // if (DESTINATION_CONTENT.equals(destination)){
if (result.size() == 0) {
logger.info("Routing {} to 'unmatched'", new Object[]{flowFile});
logger.info("No XQuery results found {}", flowFile);
session.transfer(flowFile, REL_NO_MATCH);
continue flowFileLoop;
} else if (result.size() == 1) {
final XdmItem item = result.itemAt(0);
flowFile = session.write(flowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream rawOut) throws IOException {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
writeformattedItem(item, context, out);
} catch (TransformerFactoryConfigurationError | TransformerException e) {
throw new IOException(e);
}
flowFile = session.write(flowFile, rawOut -> {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
writeformattedItem(item, context, out);
} catch (TransformerFactoryConfigurationError | TransformerException e) {
throw new IOException(e);
}
});
} else {
for (final XdmItem item : result) {
FlowFile ff = session.clone(flowFile);
ff = session.write(ff, new OutputStreamCallback() {
@Override
public void process(final OutputStream rawOut) throws IOException {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
try {
writeformattedItem(item, context, out);
} catch (TransformerFactoryConfigurationError | TransformerException e) {
throw new IOException(e);
}
ff = session.write(ff, rawOut -> {
try (final OutputStream out = new BufferedOutputStream(rawOut)) {
try {
writeformattedItem(item, context, out);
} catch (TransformerFactoryConfigurationError | TransformerException e) {
throw new IOException(e);
}
}
});
@ -381,14 +337,12 @@ public class EvaluateXQuery extends AbstractProcessor {
}
}
} catch (final SaxonApiException e) {
logger.error("failed to evaluate XQuery for {} for Property {} due to {}; routing to failure",
new Object[]{flowFile, entry.getKey(), e});
logger.error("XQuery Property [{}] processing failed", entry.getKey(), e);
session.transfer(flowFile, REL_FAILURE);
session.remove(childrenFlowFiles);
continue flowFileLoop;
} catch (TransformerFactoryConfigurationError | TransformerException | IOException e) {
logger.error("Failed to write XQuery result for {} due to {}; routing original to 'failure'",
new Object[]{flowFile, error.get()});
logger.error("XQuery Property [{}] configuration failed", entry.getKey(), e);
session.transfer(flowFile, REL_FAILURE);
session.remove(childrenFlowFiles);
continue flowFileLoop;
@ -398,18 +352,16 @@ public class EvaluateXQuery extends AbstractProcessor {
if (DESTINATION_ATTRIBUTE.equals(destination)) {
flowFile = session.putAllAttributes(flowFile, xQueryResults);
final Relationship destRel = xQueryResults.isEmpty() ? REL_NO_MATCH : REL_MATCH;
logger.info("Successfully evaluated XQueries against {} and found {} matches; routing to {}",
new Object[]{flowFile, xQueryResults.size(), destRel.getName()});
logger.info("XQuery results found [{}] for {}", xQueryResults.size(), flowFile);
session.transfer(flowFile, destRel);
session.getProvenanceReporter().modifyAttributes(flowFile);
} else { // if (DESTINATION_CONTENT.equals(destination)) {
if (!childrenFlowFiles.isEmpty()) {
logger.info("Successfully created {} new FlowFiles from {}; routing all to 'matched'",
new Object[]{childrenFlowFiles.size(), flowFile});
logger.info("XQuery results found [{}] for {} FlowFiles created [{}]", xQueryResults.size(), flowFile, childrenFlowFiles.size());
session.transfer(childrenFlowFiles, REL_MATCH);
session.remove(flowFile);
} else {
logger.info("Successfully updated content for {}; routing to 'matched'", new Object[]{flowFile});
logger.info("XQuery results found for {} content updated", flowFile);
session.transfer(flowFile, REL_MATCH);
session.getProvenanceReporter().modifyContent(flowFile);
}
@ -475,7 +427,7 @@ public class EvaluateXQuery extends AbstractProcessor {
return new ValidationResult.Builder().input(input).subject(subject).valid(error == null).explanation(error).build();
} catch (final Exception e) {
return new ValidationResult.Builder().input(input).subject(subject).valid(false)
.explanation("Unable to initialize XQuery engine due to " + e.toString()).build();
.explanation("Unable to initialize XQuery engine due to " + e).build();
}
}
}

58
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java
View File

@ -23,6 +23,7 @@ import static org.apache.nifi.flowfile.attributes.FragmentAttributes.SEGMENT_ORI
import static org.apache.nifi.flowfile.attributes.FragmentAttributes.copyAttributesToOriginal;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@ -34,8 +35,6 @@ import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.SystemResourceConsideration;
@ -59,15 +58,13 @@ import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.standard.util.XmlElementNotifier;
import org.apache.nifi.security.xml.XmlUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.sax.StandardInputSourceParser;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.Locator;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
@EventDriven
@SideEffectFree
@ -115,22 +112,6 @@ public class SplitXml extends AbstractProcessor {
private List<PropertyDescriptor> properties;
private Set<Relationship> relationships;
private static final String FEATURE_PREFIX = "http://xml.org/sax/features/";
public static final String ENABLE_NAMESPACES_FEATURE = FEATURE_PREFIX + "namespaces";
public static final String ENABLE_NAMESPACE_PREFIXES_FEATURE = FEATURE_PREFIX + "namespace-prefixes";
private static final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
static {
saxParserFactory.setNamespaceAware(true);
try {
saxParserFactory.setFeature(ENABLE_NAMESPACES_FEATURE, true);
saxParserFactory.setFeature(ENABLE_NAMESPACE_PREFIXES_FEATURE, true);
} catch (Exception e) {
final Logger staticLogger = LoggerFactory.getLogger(SplitXml.class);
staticLogger.warn("Unable to configure SAX Parser to make namespaces available", e);
}
}
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> properties = new ArrayList<>();
@ -169,7 +150,7 @@ public class SplitXml extends AbstractProcessor {
final AtomicInteger numberOfRecords = new AtomicInteger(0);
final XmlSplitterSaxParser parser = new XmlSplitterSaxParser(xmlTree -> {
FlowFile split = session.create(original);
split = session.write(split, out -> out.write(xmlTree.getBytes("UTF-8")));
split = session.write(split, out -> out.write(xmlTree.getBytes(StandardCharsets.UTF_8)));
split = session.putAttribute(split, FRAGMENT_ID.key(), fragmentIdentifier);
split = session.putAttribute(split, FRAGMENT_INDEX.key(), Integer.toString(numberOfRecords.getAndIncrement()));
split = session.putAttribute(split, SEGMENT_ORIGINAL_FILENAME.key(), split.getAttribute(CoreAttributes.FILENAME.key()));
@ -180,10 +161,11 @@ public class SplitXml extends AbstractProcessor {
session.read(original, rawIn -> {
try (final InputStream in = new java.io.BufferedInputStream(rawIn)) {
try {
final XMLReader reader = XmlUtils.createSafeSaxReader(saxParserFactory, parser);
reader.parse(new InputSource(in));
} catch (final ParserConfigurationException | SAXException e) {
logger.error("Unable to parse {} due to {}", new Object[]{original, e});
final StandardInputSourceParser inputSourceParser = new StandardInputSourceParser();
inputSourceParser.setNamespaceAware(true);
inputSourceParser.parse(new InputSource(in), parser);
} catch (final ProcessingException e) {
logger.error("Parsing failed {}", original, e);
failed.set(true);
}
}
@ -200,7 +182,7 @@ public class SplitXml extends AbstractProcessor {
final FlowFile originalToTransfer = copyAttributesToOriginal(session, original, fragmentIdentifier, numberOfRecords.get());
session.transfer(originalToTransfer, REL_ORIGINAL);
logger.info("Split {} into {} FlowFiles", new Object[]{originalToTransfer, splits.size()});
logger.info("Split {} into {} FlowFiles", originalToTransfer, splits.size());
}
}
@ -211,7 +193,7 @@ public class SplitXml extends AbstractProcessor {
private final int splitDepth;
private final StringBuilder sb = new StringBuilder(XML_PROLOGUE);
private int depth = 0;
private Map<String, String> prefixMap = new TreeMap<>();
private final Map<String, String> prefixMap = new TreeMap<>();
public XmlSplitterSaxParser(XmlElementNotifier notifier, int splitDepth) {
this.notifier = notifier;
@ -252,11 +234,11 @@ public class SplitXml extends AbstractProcessor {
}
@Override
public void endDocument() throws SAXException {
public void endDocument() {
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
public void endElement(String uri, String localName, String qName) {
// We have finished processing this element. Decrement the depth.
int newDepth = --depth;
@ -279,16 +261,16 @@ public class SplitXml extends AbstractProcessor {
}
@Override
public void endPrefixMapping(String prefix) throws SAXException {
public void endPrefixMapping(String prefix) {
prefixMap.remove(prefixToNamespace(prefix));
}
@Override
public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
public void ignorableWhitespace(char[] ch, int start, int length) {
}
@Override
public void processingInstruction(String target, String data) throws SAXException {
public void processingInstruction(String target, String data) {
}
@Override
@ -296,15 +278,15 @@ public class SplitXml extends AbstractProcessor {
}
@Override
public void skippedEntity(String name) throws SAXException {
public void skippedEntity(String name) {
}
@Override
public void startDocument() throws SAXException {
public void startDocument() {
}
@Override
public void startElement(final String uri, final String localName, final String qName, final Attributes atts) throws SAXException {
public void startElement(final String uri, final String localName, final String qName, final Attributes atts) {
// Increment the current depth because start a new XML element.
int newDepth = ++depth;
// Output the element and its attributes if it is
@ -343,7 +325,7 @@ public class SplitXml extends AbstractProcessor {
}
@Override
public void startPrefixMapping(String prefix, String uri) throws SAXException {
public void startPrefixMapping(String prefix, String uri) {
final String ns = prefixToNamespace(prefix);
prefixMap.put(ns, uri);
}

10
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java
View File

@ -48,11 +48,12 @@ import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.util.StopWatch;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import javax.xml.XMLConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Result;
@ -363,10 +364,11 @@ public class TransformXml extends AbstractProcessor {
}
private Source getSecureSource(final StreamSource streamSource) throws TransformerConfigurationException {
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
try {
final XMLStreamReader streamReader = XmlUtils.createSafeReader(streamSource);
final XMLStreamReader streamReader = provider.getStreamReader(streamSource);
return new StAXSource(streamReader);
} catch (final XMLStreamException e) {
} catch (final ProcessingException e) {
throw new TransformerConfigurationException("XSLT Source Stream Reader creation failed", e);
}
}

65
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ValidateXml.java
View File

@ -39,19 +39,22 @@ import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.security.xml.SafeXMLConfiguration;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider;
import org.apache.nifi.xml.processing.validation.StandardSchemaValidator;
import org.apache.nifi.xml.processing.validation.SchemaValidator;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.Source;
import javax.xml.transform.stax.StAXSource;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
@ -110,6 +113,10 @@ public class ValidateXml extends AbstractProcessor {
private static final String SCHEMA_LANGUAGE = "http://www.w3.org/2001/XMLSchema";
private static final SchemaValidator SCHEMA_VALIDATOR = new StandardSchemaValidator();
private static final XMLStreamReaderProvider READER_PROVIDER = new StandardXMLStreamReaderProvider();
private List<PropertyDescriptor> properties;
private Set<Relationship> relationships;
private final AtomicReference<Schema> schemaRef = new AtomicReference<>();
@ -156,41 +163,25 @@ public class ValidateXml extends AbstractProcessor {
return;
}
final Schema schema = schemaRef.get();
final Validator validator = schema == null ? null : schema.newValidator();
final ComponentLog logger = getLogger();
final boolean attributeContainsXML = context.getProperty(XML_SOURCE_ATTRIBUTE).isSet();
for (FlowFile flowFile : flowFiles) {
final AtomicBoolean valid = new AtomicBoolean(true);
final AtomicReference<Exception> exception = new AtomicReference<>(null);
SafeXMLConfiguration safeXMLConfiguration = new SafeXMLConfiguration();
safeXMLConfiguration.setValidating(false);
try {
DocumentBuilder docBuilder = safeXMLConfiguration.createDocumentBuilder();
if (attributeContainsXML) {
// If XML source attribute is set, validate attribute value
String xml = flowFile.getAttribute(context.getProperty(XML_SOURCE_ATTRIBUTE).evaluateAttributeExpressions().getValue());
ByteArrayInputStream bais = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8));
ByteArrayInputStream inputStream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8));
validate(validator, docBuilder, bais);
validate(inputStream);
} else {
// If XML source attribute is not set, validate flowfile content
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream in) throws IOException {
try {
validate(validator, docBuilder, in);
} catch (final IllegalArgumentException | SAXException e) {
valid.set(false);
exception.set(e);
}
}
});
session.read(flowFile, inputStream -> validate(inputStream));
}
} catch (final IllegalArgumentException | SAXException | ParserConfigurationException | IOException e) {
} catch (final RuntimeException e) {
valid.set(false);
exception.set(e);
}
@ -218,13 +209,23 @@ public class ValidateXml extends AbstractProcessor {
}
}
private void validate(final Validator validator, final DocumentBuilder docBuilder, final InputStream in) throws IllegalArgumentException, SAXException, IOException {
if (validator != null) {
// If schema is provided, validator will be non-null
validator.validate(new StreamSource(in));
private void validate(final InputStream in) {
final Schema schema = schemaRef.get();
if (schema == null) {
// Parse Document without schema validation
final XMLStreamReader reader = READER_PROVIDER.getStreamReader(new StreamSource(in));
try {
while (reader.hasNext()) {
reader.next();
}
} catch (final XMLStreamException e) {
throw new ProcessingException("Reading stream failed", e);
}
} else {
// Only verify that the XML is well-formed; no schema check
docBuilder.parse(in);
final XMLStreamReaderProvider readerProvider = new StandardXMLStreamReaderProvider();
final XMLStreamReader reader = readerProvider.getStreamReader(new StreamSource(in));
final Source source = new StAXSource(reader);
SCHEMA_VALIDATOR.validate(schema, source);
}
}
}

15
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/DocumentReaderCallback.java
View File

@ -18,12 +18,10 @@ package org.apache.nifi.processors.standard.util;
import java.io.IOException;
import java.io.InputStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.security.xml.XmlUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
public class DocumentReaderCallback implements InputStreamCallback {
@ -42,10 +40,11 @@ public class DocumentReaderCallback implements InputStreamCallback {
@Override
public void process(final InputStream stream) throws IOException {
try {
DocumentBuilder builder = XmlUtils.createSafeDocumentBuilder(isNamespaceAware);
document = builder.parse(stream);
} catch (ParserConfigurationException | SAXException pce) {
throw new IOException(pce.getLocalizedMessage(), pce);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(isNamespaceAware);
document = documentProvider.parse(stream);
} catch (final ProcessingException e) {
throw new IOException(e.getLocalizedMessage(), e);
}
}

34
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/xml/DocumentTypeAllowedDocumentProvider.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.standard.xml;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
/**
* Document Provider implementation that allows local Document Type Declarations
*/
public class DocumentTypeAllowedDocumentProvider extends StandardDocumentProvider {
/**
* Enable Document Type Declaration through disabling disallow configuration
*
* @return Disallow Disabled
*/
@Override
protected boolean isDisallowDocumentTypeDeclaration() {
return false;
}
}

58
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXPath.java
View File

@ -16,19 +16,19 @@
*/
package org.apache.nifi.processors.standard;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import javax.xml.xpath.XPathFactoryConfigurationException;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.Test;
import org.junit.jupiter.api.Test;
public class TestEvaluateXPath {
@ -37,7 +37,7 @@ public class TestEvaluateXPath {
private static final Path XML_SNIPPET_NONEXISTENT_DOCTYPE = Paths.get("src/test/resources/TestXml/xml-snippet-external-doctype.xml");
@Test
public void testAsAttribute() throws XPathFactoryConfigurationException, IOException {
public void testAsAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_ATTRIBUTE);
testRunner.setProperty("xpath.result1", "/");
@ -53,7 +53,7 @@ public class TestEvaluateXPath {
}
@Test
public void testCheckIfElementExists() throws XPathFactoryConfigurationException, IOException {
public void testCheckIfElementExists() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_ATTRIBUTE);
testRunner.setProperty("xpath.result1", "/");
@ -71,7 +71,7 @@ public class TestEvaluateXPath {
}
@Test
public void testUnmatched() throws XPathFactoryConfigurationException, IOException {
public void testUnmatched() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty("xpath.result.exist.2", "/*:bundle/node2");
@ -83,7 +83,7 @@ public class TestEvaluateXPath {
testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_NO_MATCH).get(0).assertContentEquals(XML_SNIPPET);
}
@Test(expected = java.lang.AssertionError.class)
@Test
public void testMultipleXPathForContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
@ -92,11 +92,12 @@ public class TestEvaluateXPath {
testRunner.setProperty("some.property.2", "/*:bundle/node/subNode[2]");
testRunner.enqueue(XML_SNIPPET);
testRunner.run();
assertThrows(AssertionError.class, testRunner::run);
}
@Test
public void testWriteToContent() throws XPathFactoryConfigurationException, IOException {
public void testWriteToContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty("some.property", "/*:bundle/node/subNode[1]");
@ -107,13 +108,13 @@ public class TestEvaluateXPath {
testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
final String outXml = new String(outData, StandardCharsets.UTF_8);
assertTrue(outXml.contains("subNode"));
assertTrue(outXml.contains("Hello"));
}
@Test
public void testFailureIfContentMatchesMultipleNodes() throws XPathFactoryConfigurationException, IOException {
public void testFailureIfContentMatchesMultipleNodes() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty("some.property", "/*:bundle/node/subNode");
@ -125,7 +126,7 @@ public class TestEvaluateXPath {
}
@Test
public void testWriteStringToContent() throws XPathFactoryConfigurationException, IOException {
public void testWriteStringToContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING);
@ -136,13 +137,11 @@ public class TestEvaluateXPath {
testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("Hello"));
out.assertContentEquals("Hello");
}
@Test
public void testWriteNodeSetToAttribute() throws XPathFactoryConfigurationException, IOException {
public void testWriteNodeSetToAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_ATTRIBUTE);
testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_NODESET);
@ -159,7 +158,7 @@ public class TestEvaluateXPath {
}
@Test
public void testSuccessForEmbeddedDocTypeValidation() throws XPathFactoryConfigurationException, IOException {
public void testSuccessForEmbeddedDocTypeValidation() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING);
@ -171,13 +170,11 @@ public class TestEvaluateXPath {
testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("Hello"));
out.assertContentEquals("Hello");
}
@Test
public void testSuccessForEmbeddedDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException {
public void testFailureForEmbeddedDocTypeValidationDisabled() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING);
@ -187,15 +184,11 @@ public class TestEvaluateXPath {
testRunner.enqueue(XML_SNIPPET_EMBEDDED_DOCTYPE);
testRunner.run();
testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("Hello"));
testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_FAILURE, 1);
}
@Test
public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws XPathFactoryConfigurationException, IOException {
public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING);
@ -208,7 +201,7 @@ public class TestEvaluateXPath {
}
@Test
public void testSuccessForExternalDocTypeWithDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException {
public void testFailureForExternalDocTypeWithDocTypeValidationDisabled() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath());
testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING);
@ -218,11 +211,6 @@ public class TestEvaluateXPath {
testRunner.enqueue(XML_SNIPPET_NONEXISTENT_DOCTYPE);
testRunner.run();
testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("Hello"));
testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_FAILURE, 1);
}
}

238
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXQuery.java
View File

@ -16,15 +16,18 @@
*/
package org.apache.nifi.processors.standard;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -32,13 +35,12 @@ import java.util.Map.Entry;
import java.util.Properties;
import javax.xml.transform.OutputKeys;
import javax.xml.xpath.XPathFactoryConfigurationException;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.Test;
import org.junit.jupiter.api.Test;
public class TestEvaluateXQuery {
@ -52,16 +54,15 @@ public class TestEvaluateXQuery {
private static final boolean[] booleans = {true, false};
@Test
public void testSetTransformerProperties() throws Exception {
for (int i = 0; i < methods.length; i++) {
for (int j = 0; j < booleans.length; j++) {
for (int k = 0; k < booleans.length; k++) {
Properties props = EvaluateXQuery.getTransformerProperties(methods[i], booleans[j], booleans[k]);
public void testSetTransformerProperties() {
for (final String method : methods) {
for (final boolean indent : booleans) {
for (final boolean omitDeclaration : booleans) {
Properties props = EvaluateXQuery.getTransformerProperties(method, indent, omitDeclaration);
assertEquals(3, props.size());
assertEquals(methods[i], props.getProperty(OutputKeys.METHOD));
assertEquals(booleans[j] ? "yes" : "no", props.getProperty(OutputKeys.INDENT));
assertEquals(booleans[k] ? "yes" : "no", props.getProperty(OutputKeys.OMIT_XML_DECLARATION));
assertEquals(method, props.getProperty(OutputKeys.METHOD));
assertEquals(indent ? "yes" : "no", props.getProperty(OutputKeys.INDENT));
assertEquals(omitDeclaration ? "yes" : "no", props.getProperty(OutputKeys.OMIT_XML_DECLARATION));
}
}
}
@ -75,20 +76,20 @@ public class TestEvaluateXQuery {
final String singleElementNodeQuery = "//fruit[1]";
final String singleTextNodeQuery = "//fruit[1]/name/text()";
for (int i = 0; i < methods.length; i++) {
for (int j = 0; j < booleans.length; j++) {
for (int k = 0; k < booleans.length; k++) {
formattedResults = getFormattedResult(XML_SNIPPET, atomicQuery, methods[i], booleans[j], booleans[k]);
for (final String method : methods) {
for (final boolean indent : booleans) {
for (final boolean omitDeclaration : booleans) {
formattedResults = getFormattedResult(XML_SNIPPET, atomicQuery, method, indent, omitDeclaration);
assertEquals(1, formattedResults.size());
assertEquals("7", formattedResults.get(0));
}
}
}
for (int i = 0; i < methods.length; i++) {
for (int j = 0; j < booleans.length; j++) {
for (int k = 0; k < booleans.length; k++) {
formattedResults = getFormattedResult(XML_SNIPPET, singleTextNodeQuery, methods[i], booleans[j], booleans[k]);
for (final String method : methods) {
for (final boolean indent : booleans) {
for (final boolean omitDeclaration : booleans) {
formattedResults = getFormattedResult(XML_SNIPPET, singleTextNodeQuery, method, indent, omitDeclaration);
assertEquals(1, formattedResults.size());
assertEquals("apple", formattedResults.get(0));
}
@ -161,7 +162,6 @@ public class TestEvaluateXQuery {
List<MockFlowFile> resultFlowFiles;
List<String> resultStrings = new ArrayList<>();
runnerProps.clear();
runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_CONTENT);
runnerProps.put(EvaluateXQuery.XML_OUTPUT_METHOD.getName(), method);
runnerProps.put(EvaluateXQuery.XML_OUTPUT_INDENT.getName(), Boolean.toString(indent));
@ -169,45 +169,44 @@ public class TestEvaluateXQuery {
runnerProps.put("xquery", xQuery);
resultFlowFiles = runXquery(xml, runnerProps);
for (int i = 0; i < resultFlowFiles.size(); i++) {
final MockFlowFile out = resultFlowFiles.get(i);
final byte[] outData = out.toByteArray();
final String outXml = new String(outData, "UTF-8");
for (final MockFlowFile flowFile : resultFlowFiles) {
final byte[] outData = flowFile.toByteArray();
final String outXml = new String(outData, StandardCharsets.UTF_8);
resultStrings.add(outXml);
}
return resultStrings;
}
@Test(expected = java.lang.AssertionError.class)
public void testBadXQuery() throws Exception {
doXqueryTest(XML_SNIPPET, "counttttttt(*:fruitbasket/fruit)", Arrays.asList("7"));
@Test
public void testBadXQuery() {
assertThrows(AssertionError.class, () -> doXqueryTest(XML_SNIPPET, "counttttttt(*:fruitbasket/fruit)", Collections.singletonList("7")));
}
@Test
public void testXQueries() throws Exception {
/* count matches */
doXqueryTest(XML_SNIPPET, "count(*:fruitbasket/fruit)", Arrays.asList("7"));
doXqueryTest(XML_SNIPPET, "count(//fruit)", Arrays.asList("7"));
doXqueryTest(XML_SNIPPET, "count(*:fruitbasket/fruit)", Collections.singletonList("7"));
doXqueryTest(XML_SNIPPET, "count(//fruit)", Collections.singletonList("7"));
/* Using a namespace */
doXqueryTest(XML_SNIPPET, "declare namespace fb = \"http://namespace/1\"; count(fb:fruitbasket/fruit)", Arrays.asList("7"));
doXqueryTest(XML_SNIPPET, "declare namespace fb = \"http://namespace/1\"; count(fb:fruitbasket/fruit)", Collections.singletonList("7"));
/* determine if node exists */
doXqueryTest(XML_SNIPPET, "boolean(//fruit[1])", Arrays.asList("true"));
doXqueryTest(XML_SNIPPET, "boolean(//fruit[100])", Arrays.asList("false"));
doXqueryTest(XML_SNIPPET, "boolean(//fruit[1])", Collections.singletonList("true"));
doXqueryTest(XML_SNIPPET, "boolean(//fruit[100])", Collections.singletonList("false"));
/* XML first match */
doXqueryTest(XML_SNIPPET, "//fruit[1]", Arrays.asList(
doXqueryTest(XML_SNIPPET, "//fruit[1]", Collections.singletonList(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><fruit xmlns:ns=\"http://namespace/1\" taste=\"crisp\"><!-- Apples are my favorite --><name>apple</name><color>red</color></fruit>"));
/* XML last match */
doXqueryTest(XML_SNIPPET, "//fruit[count(//fruit)]", Arrays.asList(
doXqueryTest(XML_SNIPPET, "//fruit[count(//fruit)]", Collections.singletonList(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><fruit xmlns:ns=\"http://namespace/1\"><name>none</name><color/></fruit>"));
/* XML first match wrapped */
doXqueryTest(XML_SNIPPET, "<wrap>{//fruit[1]}</wrap>", Arrays.asList(
doXqueryTest(XML_SNIPPET, "<wrap>{//fruit[1]}</wrap>", Collections.singletonList(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><wrap><fruit xmlns:ns=\"http://namespace/1\" taste=\"crisp\"><!-- Apples are my favorite --><name>apple</name><color>red</color></fruit></wrap>"));
/* XML all matches (multiple results) */
@ -221,7 +220,7 @@ public class TestEvaluateXQuery {
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><fruit xmlns:ns=\"http://namespace/1\"><name>none</name><color/></fruit>"));
/* XML all matches wrapped (one result)*/
doXqueryTest(XML_SNIPPET, "<wrap>{//fruit}</wrap>", Arrays.asList(
doXqueryTest(XML_SNIPPET, "<wrap>{//fruit}</wrap>", Collections.singletonList(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ "<wrap>"
+ "<fruit xmlns:ns=\"http://namespace/1\" taste=\"crisp\"><!-- Apples are my favorite --><name>apple</name><color>red</color></fruit>"
@ -237,18 +236,18 @@ public class TestEvaluateXQuery {
doXqueryTest(XML_SNIPPET, "for $x in //fruit return $x/name/text()", Arrays.asList(fruitNames));
/* String first match fruit name (XPath)*/
doXqueryTest(XML_SNIPPET, "//fruit[1]/name/text()", Arrays.asList("apple"));
doXqueryTest(XML_SNIPPET, "//fruit[1]/name/text()", Collections.singletonList("apple"));
/* String first match fruit color (XPath)*/
doXqueryTest(XML_SNIPPET, "//fruit[1]/color/text()", Arrays.asList("red"));
doXqueryTest(XML_SNIPPET, "//fruit[1]/color/text()", Collections.singletonList("red"));
/* String first match fruit name (XQuery)*/
doXqueryTest(XML_SNIPPET, "for $x in //fruit[1] return string-join(($x/name/text() , $x/color/text()), ' - ')",
Arrays.asList("apple - red"));
Collections.singletonList("apple - red"));
/* String first match fruit & color (one result)*/
doXqueryTest(XML_SNIPPET, "for $x in //fruit[1] return string-join(($x/name/text() , $x/color/text()), ' - ')",
Arrays.asList("apple - red"));
Collections.singletonList("apple - red"));
/* String all matches fruit & color (multiple results)*/
doXqueryTest(XML_SNIPPET, "for $x in //fruit return string-join(($x/name/text() , $x/color/text()), ' - ')",
@ -263,7 +262,7 @@ public class TestEvaluateXQuery {
/* String all matches fruit & color (single, newline delimited result)*/
doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit return string-join(($x/name/text() , $x/color/text()), ' - ')) return $y), '\n')",
Arrays.asList(
Collections.singletonList(
"apple - red\n"
+ "apple - green\n"
+ "banana - yellow\n"
@ -274,7 +273,7 @@ public class TestEvaluateXQuery {
/* String all matches fruit & color using "let" (single, newline delimited result)*/
doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit let $d := string-join(($x/name/text() , $x/color/text()), ' - ') return $d) return $y), '\n')",
Arrays.asList(
Collections.singletonList(
"apple - red\n"
+ "apple - green\n"
+ "banana - yellow\n"
@ -285,25 +284,25 @@ public class TestEvaluateXQuery {
/* String all matches name only, comma delimited (one result)*/
doXqueryTest(XML_SNIPPET, "string-join((for $x in //fruit return $x/name/text()), ', ')",
Arrays.asList("apple, apple, banana, orange, blueberry, raspberry, none"));
Collections.singletonList("apple, apple, banana, orange, blueberry, raspberry, none"));
/* String all matches color and name, comma delimited (one result)*/
doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit return string-join(($x/color/text() , $x/name/text()), ' ')) return $y), ', ')",
Arrays.asList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none"));
Collections.singletonList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none"));
/* String all matches color and name, comma delimited using let(one result)*/
doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit let $d := string-join(($x/color/text() , $x/name/text()), ' ') return $d) return $y), ', ')",
Arrays.asList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none"));
Collections.singletonList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none"));
/* Query for attribute */
doXqueryTest(XML_SNIPPET, "string(//fruit[1]/@taste)", Arrays.asList("crisp"));
doXqueryTest(XML_SNIPPET, "string(//fruit[1]/@taste)", Collections.singletonList("crisp"));
/* Query for comment */
doXqueryTest(XML_SNIPPET, "//fruit/comment()", Arrays.asList(" Apples are my favorite "));
doXqueryTest(XML_SNIPPET, "//fruit/comment()", Collections.singletonList(" Apples are my favorite "));
/* Query for processing instruction */
doXqueryTest(XML_SNIPPET, "//processing-instruction()[name()='xml-stylesheet']", Arrays.asList("type=\"text/xsl\" href=\"foo.xsl\""));
doXqueryTest(XML_SNIPPET, "//processing-instruction()[name()='xml-stylesheet']", Collections.singletonList("type=\"text/xsl\" href=\"foo.xsl\""));
}
@ -313,50 +312,44 @@ public class TestEvaluateXQuery {
List<MockFlowFile> resultFlowFiles;
// test read from content, write to attribute
{
runnerProps.clear();
runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_ATTRIBUTE);
runnerProps.put("xquery", xQuery);
resultFlowFiles = runXquery(xml, runnerProps);
runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_ATTRIBUTE);
runnerProps.put("xquery", xQuery);
resultFlowFiles = runXquery(xml, runnerProps);
assertEquals(1, resultFlowFiles.size());
assertEquals(1, resultFlowFiles.size());
final MockFlowFile out = resultFlowFiles.get(0);
final MockFlowFile out = resultFlowFiles.get(0);
for (int i = 0; i < expectedResults.size(); i++) {
String key = "xquery";
if (expectedResults.size() > 1) {
key += "." + ((int) i + 1);
}
final String actual = out.getAttribute(key).replaceAll(">\\s+<", "><");
final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><");
assertEquals(expected, actual);
for (int i = 0; i < expectedResults.size(); i++) {
String key = "xquery";
if (expectedResults.size() > 1) {
key += "." + (i + 1);
}
final String actual = out.getAttribute(key).replaceAll(">\\s+<", "><");
final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><");
assertEquals(expected, actual);
}
// test read from content, write to content
{
runnerProps.clear();
runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_CONTENT);
runnerProps.put("xquery", xQuery);
resultFlowFiles = runXquery(xml, runnerProps);
runnerProps.clear();
runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_CONTENT);
runnerProps.put("xquery", xQuery);
resultFlowFiles = runXquery(xml, runnerProps);
assertEquals(expectedResults.size(), resultFlowFiles.size());
assertEquals(expectedResults.size(), resultFlowFiles.size());
for (int i = 0; i < resultFlowFiles.size(); i++) {
for (int i = 0; i < resultFlowFiles.size(); i++) {
final MockFlowFile out = resultFlowFiles.get(i);
final byte[] outData = out.toByteArray();
final String outXml = new String(outData, "UTF-8").replaceAll(">\\s+<", "><");
final String actual = outXml;
final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><");
assertEquals(expected, actual);
}
final MockFlowFile resultFlowFile = resultFlowFiles.get(i);
final byte[] outData = resultFlowFile.toByteArray();
final String outXml = new String(outData, StandardCharsets.UTF_8).replaceAll(">\\s+<", "><");
final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><");
assertEquals(expected, outXml);
}
}
private List<MockFlowFile> runXquery(Path xml, Map<String, String> runnerProps) throws Exception {
return runXquery(xml, runnerProps, new HashMap<String, String>());
return runXquery(xml, runnerProps, new HashMap<>());
}
private List<MockFlowFile> runXquery(Path xml, Map<String, String> runnerProps, Map<String, String> flowFileAttributes) throws Exception {
@ -376,7 +369,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testRootPath() throws XPathFactoryConfigurationException, IOException {
public void testRootPath() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("xquery.result1", "/");
@ -387,13 +380,13 @@ public class TestEvaluateXQuery {
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
final String attributeString = out.getAttribute("xquery.result1").replaceAll(">\\s+<", "><");
final String xmlSnippetString = new String(Files.readAllBytes(XML_SNIPPET), "UTF-8").replaceAll(">\\s+<", "><");
final String xmlSnippetString = new String(Files.readAllBytes(XML_SNIPPET), StandardCharsets.UTF_8).replaceAll(">\\s+<", "><");
assertEquals(xmlSnippetString, attributeString);
}
@Test
public void testCheckIfElementExists() throws XPathFactoryConfigurationException, IOException {
public void testCheckIfElementExists() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("xquery.result.exist.1", "boolean(/*:fruitbasket/fruit[1])");
@ -409,7 +402,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testUnmatchedContent() throws XPathFactoryConfigurationException, IOException {
public void testUnmatchedContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty("xquery.result.exist.2", "/*:fruitbasket/node2");
@ -422,7 +415,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testUnmatchedAttribute() throws XPathFactoryConfigurationException, IOException {
public void testUnmatchedAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("xquery.result.exist.2", "/*:fruitbasket/node2");
@ -437,7 +430,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testNoXQueryAttribute() throws XPathFactoryConfigurationException, IOException {
public void testNoXQueryAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
@ -448,17 +441,17 @@ public class TestEvaluateXQuery {
testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_NO_MATCH).get(0).assertContentEquals(XML_SNIPPET);
}
@Test(expected = java.lang.AssertionError.class)
public void testNoXQueryContent() throws XPathFactoryConfigurationException, IOException {
@Test
public void testNoXQueryContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.enqueue(XML_SNIPPET);
testRunner.run();
assertThrows(AssertionError.class, testRunner::run);
}
@Test
public void testOneMatchOneUnmatchAttribute() throws XPathFactoryConfigurationException, IOException {
public void testOneMatchOneUnmatchAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("some.property", "//fruit/name/text()");
@ -472,7 +465,7 @@ public class TestEvaluateXQuery {
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
for (int i = 0; i < fruitNames.length; i++) {
final String outXml = out.getAttribute("some.property." + ((int) i + 1));
final String outXml = out.getAttribute("some.property." + (i + 1));
assertEquals(fruitNames[i], outXml.trim());
}
@ -481,7 +474,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testMatchedEmptyStringAttribute() throws XPathFactoryConfigurationException, IOException {
public void testMatchedEmptyStringAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("xquery.result.exist.2", "/*:fruitbasket/*[name='none']/color/text()");
@ -496,7 +489,7 @@ public class TestEvaluateXQuery {
testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_NO_MATCH).get(0).assertContentEquals(XML_SNIPPET);
}
@Test(expected = java.lang.AssertionError.class)
@Test
public void testMultipleXPathForContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
@ -504,11 +497,11 @@ public class TestEvaluateXQuery {
testRunner.setProperty("some.property.2", "/*:fruitbasket/fruit[2]");
testRunner.enqueue(XML_SNIPPET);
testRunner.run();
assertThrows(AssertionError.class, testRunner::run);
}
@Test
public void testWriteStringToAttribute() throws XPathFactoryConfigurationException, IOException {
public void testWriteStringToAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("xquery.result2", "/*:fruitbasket/fruit[1]/name/text()");
@ -523,7 +516,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testWriteStringToContent() throws XPathFactoryConfigurationException, IOException {
public void testWriteStringToContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name/text()");
@ -533,13 +526,11 @@ public class TestEvaluateXQuery {
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("apple"));
out.assertContentEquals("apple");
}
@Test
public void testWriteXmlToAttribute() throws XPathFactoryConfigurationException, IOException {
public void testWriteXmlToAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name");
@ -555,7 +546,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testWriteXmlToContent() throws XPathFactoryConfigurationException, IOException {
public void testWriteXmlToContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name");
@ -566,12 +557,12 @@ public class TestEvaluateXQuery {
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
final String outXml = new String(outData, StandardCharsets.UTF_8);
assertTrue(outXml.contains("<name xmlns:ns=\"http://namespace/1\">apple</name>"));
}
@Test
public void testMatchesMultipleStringContent() throws XPathFactoryConfigurationException, IOException {
public void testMatchesMultipleStringContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty("some.property", "//fruit/name/text()");
@ -585,14 +576,12 @@ public class TestEvaluateXQuery {
for (int i = 0; i < flowFilesForRelMatch.size(); i++) {
final MockFlowFile out = flowFilesForRelMatch.get(i);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertEquals(fruitNames[i], outXml.trim());
out.assertContentEquals(fruitNames[i]);
}
}
@Test
public void testMatchesMultipleStringAttribute() throws XPathFactoryConfigurationException, IOException {
public void testMatchesMultipleStringAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("some.property", "//fruit/name/text()");
@ -605,14 +594,14 @@ public class TestEvaluateXQuery {
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
for (int i = 0; i < fruitNames.length; i++) {
final String outXml = out.getAttribute("some.property." + ((int) i + 1));
final String outXml = out.getAttribute("some.property." + (i + 1));
assertEquals(fruitNames[i], outXml.trim());
}
testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0).assertContentEquals(XML_SNIPPET);
}
@Test
public void testMatchesMultipleXmlContent() throws XPathFactoryConfigurationException, IOException {
public void testMatchesMultipleXmlContent() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty("some.property", "//fruit/name");
@ -627,14 +616,14 @@ public class TestEvaluateXQuery {
final MockFlowFile out = flowFilesForRelMatch.get(i);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
final String outXml = new String(outData, StandardCharsets.UTF_8);
String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><name xmlns:ns=\"http://namespace/1\">" + fruitNames[i] + "</name>";
assertEquals(expectedXml, outXml.trim());
}
}
@Test
public void testMatchesMultipleXmlAttribute() throws XPathFactoryConfigurationException, IOException {
public void testMatchesMultipleXmlAttribute() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE);
testRunner.setProperty("some.property", "//fruit/name");
@ -647,7 +636,7 @@ public class TestEvaluateXQuery {
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
for (int i = 0; i < fruitNames.length; i++) {
final String outXml = out.getAttribute("some.property." + ((int) i + 1));
final String outXml = out.getAttribute("some.property." + (i + 1));
String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><name xmlns:ns=\"http://namespace/1\">" + fruitNames[i] + "</name>";
assertEquals(expectedXml, outXml.trim());
}
@ -655,7 +644,7 @@ public class TestEvaluateXQuery {
}
@Test
public void testSuccessForEmbeddedDocTypeValidation() throws XPathFactoryConfigurationException, IOException {
public void testSuccessForEmbeddedDocTypeValidation() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXQuery.VALIDATE_DTD, "true");
@ -666,13 +655,11 @@ public class TestEvaluateXQuery {
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("Hello"));
out.assertContentEquals("Hello");
}
@Test
public void testSuccessForEmbeddedDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException {
public void testFailureForEmbeddedDocTypeValidationDisabled() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXQuery.VALIDATE_DTD, "false");
@ -681,16 +668,12 @@ public class TestEvaluateXQuery {
testRunner.enqueue(XML_SNIPPET_EMBEDDED_DOCTYPE);
testRunner.run();
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("Hello"));
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_FAILURE, 1);
}
@Test
public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws XPathFactoryConfigurationException, IOException {
public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty("some.property", "/*:bundle/node/subNode[1]/value/text()");
@ -701,9 +684,8 @@ public class TestEvaluateXQuery {
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_FAILURE, 1);
}
@Test
public void testSuccessForExternalDocTypeWithDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException {
public void testFailureForExternalDocTypeWithDocTypeValidationDisabled() throws IOException {
final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery());
testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT);
testRunner.setProperty(EvaluateXQuery.VALIDATE_DTD, "false");
@ -712,10 +694,6 @@ public class TestEvaluateXQuery {
testRunner.enqueue(XML_SNIPPET_NONEXISTENT_DOCTYPE);
testRunner.run();
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1);
final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0);
final byte[] outData = testRunner.getContentAsByteArray(out);
final String outXml = new String(outData, "UTF-8");
assertTrue(outXml.trim().equals("Hello"));
testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_FAILURE, 1);
}
}

5
nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml
View File

@ -176,11 +176,6 @@
<version>${nifi.groovy.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>

2
nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/XMLFileLookupService.java
View File

@ -19,7 +19,7 @@ package org.apache.nifi.lookup;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.lookup.configuration2.CommonsConfigurationLookupService;
import org.apache.nifi.security.xml.SafeXMLConfiguration;
import org.apache.nifi.lookup.configuration2.SafeXMLConfiguration;
@Tags({"lookup", "cache", "enrich", "join", "xml", "reloadable", "key", "value"})

1
nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/CommonsConfigurationLookupService.java
View File

@ -36,7 +36,6 @@ import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.lookup.LookupFailureException;
import org.apache.nifi.lookup.StringLookupService;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.security.xml.XXEValidator;
import java.io.File;
import java.lang.reflect.ParameterizedType;

6
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/SafeXMLConfiguration.java → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/SafeXMLConfiguration.java
View File

@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.xml;
package org.apache.nifi.lookup.configuration2;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
@ -45,7 +46,7 @@ public class SafeXMLConfiguration extends XMLConfiguration {
private static final String W3C_XML_SCHEMA =
"http://www.w3.org/2001/XMLSchema";
// These features are used to disable processing external entities in the DocumentBuilderFactory to protect against XXE attacks
// These features are used to disable processing external entities to protect against XXE attacks
private static final String DISALLOW_DOCTYPES = "http://apache.org/xml/features/disallow-doctype-decl";
private static final String ALLOW_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities";
private static final String ALLOW_EXTERNAL_PARAM_ENTITIES = "http://xml.org/sax/features/external-parameter-entities";
@ -90,6 +91,7 @@ public class SafeXMLConfiguration extends XMLConfiguration {
}
// Disable DTDs and external entities to protect against XXE
factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
factory.setAttribute(DISALLOW_DOCTYPES, true);
factory.setAttribute(ALLOW_EXTERNAL_GENERAL_ENTITIES, false);
factory.setAttribute(ALLOW_EXTERNAL_PARAM_ENTITIES, false);

2
nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XXEValidator.java → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/XXEValidator.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security.xml;
package org.apache.nifi.lookup.configuration2;
import java.io.BufferedReader;
import java.io.IOException;

3
nifi-commons/nifi-security-utils/src/test/java/org/apache/nifi/security/XXEValidatorTest.java → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/java/org/apache/nifi/lookup/configuration2/XXEValidatorTest.java
View File

@ -14,11 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.security;
package org.apache.nifi.lookup.configuration2;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.security.xml.XXEValidator;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

0
nifi-commons/nifi-security-utils/src/test/resources/local_xxe_file.xml → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/local_xxe_file.xml
View File

0
nifi-commons/nifi-security-utils/src/test/resources/multiline_xxe_file.xml → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/multiline_xxe_file.xml
View File

0
nifi-commons/nifi-security-utils/src/test/resources/no_xxe.xml → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/no_xxe.xml
View File

0
nifi-commons/nifi-security-utils/src/test/resources/remote_xxe_file.xml → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/remote_xxe_file.xml
View File

0
nifi-commons/nifi-security-utils/src/test/resources/whitespace_xxe_file.xml → nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/whitespace_xxe_file.xml
View File

5
nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml
View File

@ -51,6 +51,11 @@
<artifactId>nifi-syslog-utils</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-xml-processing</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>

18
nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/windowsevent/WindowsEventLogRecordReader.java
View File

@ -32,14 +32,17 @@ import org.apache.nifi.serialization.record.util.DataTypeUtils;
import org.apache.nifi.stream.io.NonCloseableInputStream;
import org.apache.nifi.util.StringUtils;
import org.apache.nifi.xml.inference.XmlSchemaInference;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.Attribute;
import javax.xml.stream.events.Characters;
import javax.xml.stream.events.StartElement;
import javax.xml.stream.events.XMLEvent;
import javax.xml.transform.stream.StreamSource;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -144,18 +147,13 @@ public class WindowsEventLogRecordReader implements RecordReader {
LAZY_TIMESTAMP_FORMAT = () -> tsf;
final FilterInputStream inputStream;
final XMLInputFactory xmlInputFactory;
final XMLEventReaderProvider provider = new StandardXMLEventReaderProvider();
try {
xmlInputFactory = XMLInputFactory.newInstance();
// Avoid XXE Vulnerabilities
xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false);
inputStream = new NonCloseableInputStream(in);
inputStream.mark(Integer.MAX_VALUE);
xmlEventReader = xmlInputFactory.createXMLEventReader(inputStream);
xmlEventReader = provider.getEventReader(new StreamSource(inputStream));
xmlSchemaInference = new XmlSchemaInference(new TimeValueInference(dateFormat, timeFormat, timestampFormat));
} catch (XMLStreamException e) {
} catch (final ProcessingException e) {
throw new MalformedRecordException("Error creating XML Event reader from FlowFile input stream", e);
}
@ -169,7 +167,7 @@ public class WindowsEventLogRecordReader implements RecordReader {
try {
// Restart the XML event stream and advance to the first Event tag
inputStream.reset();
xmlEventReader = xmlInputFactory.createXMLEventReader(inputStream);
xmlEventReader = provider.getEventReader(new StreamSource(inputStream));
if (isArray) {
skipToNextStartTag();
}

Some files were not shown because too many files have changed in this diff Show More