From 15f7590f7ab27c16166200028d86f3ebd2974092 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Tue, 12 Apr 2022 15:32:27 -0500 Subject: [PATCH] NIFI-9901 Added nifi-xml-processing to nifi-commons - Refactored XML parsing to use providers from nifi-xml-processing - Configured spotbugs-maven-plugin with findsecbugs-plugin in nifi-xml-processing - Disabled Validate DTD in default configuration for EvaluateXPath and EvaluateXQuery - Replaced configuration of DocumentBuilder and streaming XML Readers with shared components - Removed XML utilities from nifi-security-utils - Moved Commons Configuration classes to nifi-lookup-services This closes #5962 Signed-off-by: Paul Grey --- minifi/minifi-bootstrap/pom.xml | 4 + .../bootstrap/util/ConfigTransformer.java | 19 +- .../bootstrap/util/ConfigTransformerTest.java | 39 +-- minifi/pom.xml | 5 + nifi-bootstrap/pom.xml | 5 + .../bootstrap/NotificationServiceManager.java | 36 +-- nifi-commons/nifi-security-utils/pom.xml | 61 ----- .../apache/nifi/security/xml/XmlUtils.java | 102 -------- .../nifi/security/xml/XmlUtilsTest.groovy | 100 -------- nifi-commons/nifi-single-user-utils/pom.xml | 5 + .../StandardLoginCredentialsWriter.java | 12 +- nifi-commons/nifi-site-to-site-client/pom.xml | 5 + .../cluster/ClusterNodeInformation.java | 12 +- nifi-commons/nifi-xml-processing/pom.xml | 55 ++++ .../xml/processing/ProcessingException.java | 32 +++ .../xml/processing/ProcessingFeature.java | 53 ++++ .../processing/parsers/DocumentProvider.java | 41 +++ .../parsers/StandardDocumentProvider.java | 126 ++++++++++ .../xml/processing/sax/InputSourceParser.java | 33 +++ .../sax/StandardInputSourceParser.java | 90 +++++++ .../StandardXMLEventReaderProvider.java | 51 ++++ .../StandardXMLStreamReaderProvider.java | 51 ++++ .../stream/XMLEventReaderProvider.java | 33 +++ .../stream/XMLStreamReaderProvider.java | 33 +++ .../validation/SchemaValidator.java | 33 +++ .../validation/StandardSchemaValidator.java | 59 +++++ .../nifi/xml/processing/ResourceProvider.java | 65 +++++ .../parsers/StandardDocumentProviderTest.java | 108 ++++++++ .../sax/StandardInputSourceParserTest.java | 85 +++++++ .../StandardXMLEventReaderProviderTest.java | 67 +++++ .../StandardXMLStreamReaderProviderTest.java | 67 +++++ .../StandardSchemaValidatorTest.java | 68 +++++ .../standard-document-doctype-entity.xml | 16 ++ .../resources/standard-document-doctype.xml | 16 ++ .../src/test/resources/standard-document.xml | 16 ++ ...dard-namespace-document-doctype-entity.xml | 16 ++ .../resources/standard-namespace-document.xml | 16 ++ .../src/test/resources/standard-schema.xsd | 18 ++ nifi-commons/pom.xml | 1 + .../AbstractPolicyBasedAuthorizer.java | 19 +- .../nifi-evtx-processors/pom.xml | 6 + .../nifi/processors/evtx/ParseEvtxTest.java | 26 +- .../authorization/AuthorizerFactoryBean.java | 10 +- .../nifi-framework/nifi-documentation/pom.xml | 5 + .../nifi/documentation/html/XmlValidator.java | 28 +-- .../nifi-file-authorizer/pom.xml | 5 + .../FileAccessPolicyProvider.java | 23 +- .../authorization/FileUserGroupProvider.java | 26 +- .../nifi/authorization/FlowParserTest.groovy | 66 ----- .../src/test/resources/flow-with-xxe.xml.gz | Bin 794 -> 0 bytes .../pom.xml | 5 + .../StandardManagedAuthorizer.java | 15 +- .../nifi-framework-cluster-protocol/pom.xml | 10 + .../cluster/protocol/HeartbeatPayload.java | 11 +- .../cluster/protocol/StandardDataFlow.java | 26 +- .../protocol/jaxb/JaxbProtocolContext.java | 11 +- .../nifi-framework-components/pom.xml | 10 + .../config/StateManagerConfiguration.java | 18 +- .../nifi-framework-core/pom.xml | 5 + .../apache/nifi/cluster/BulletinsPayload.java | 12 +- .../serialization/StandardFlowSerializer.java | 17 +- .../service/ControllerServiceLoader.java | 15 +- .../nifi/fingerprint/FingerprintFactory.java | 35 +-- .../StandardSnippetDeserializer.java | 12 +- .../persistence/TemplateDeserializer.java | 11 +- .../java/org/apache/nifi/util/FlowParser.java | 28 ++- .../fingerprint/FingerprintFactoryTest.java | 74 ++---- .../persistence/TemplateSerializerTest.java | 7 +- .../nifi-web/nifi-web-api/pom.xml | 4 + .../apache/nifi/audit/ProcessorAuditor.java | 18 +- .../nifi/web/api/ProcessGroupResource.java | 8 +- .../LoginIdentityProviderFactoryBean.java | 6 +- .../nifi-framework-bundle/pom.xml | 5 + .../nifi-ranger-plugin/pom.xml | 2 +- .../ManagedRangerAuthorizer.java | 18 +- .../nifi-single-user-iaa-providers/pom.xml | 5 + .../single/user/SingleUserAuthorizer.java | 10 +- .../nifi-standard-processors/pom.xml | 5 + .../processors/standard/EvaluateXPath.java | 161 +++++------- .../processors/standard/EvaluateXQuery.java | 132 ++++------ .../nifi/processors/standard/SplitXml.java | 58 ++--- .../processors/standard/TransformXml.java | 10 +- .../nifi/processors/standard/ValidateXml.java | 65 ++--- .../standard/util/DocumentReaderCallback.java | 15 +- .../DocumentTypeAllowedDocumentProvider.java | 34 +++ .../standard/TestEvaluateXPath.java | 58 ++--- .../standard/TestEvaluateXQuery.java | 238 ++++++++---------- .../nifi-lookup-services/pom.xml | 5 - .../nifi/lookup/XMLFileLookupService.java | 2 +- .../CommonsConfigurationLookupService.java | 1 - .../configuration2}/SafeXMLConfiguration.java | 6 +- .../lookup/configuration2}/XXEValidator.java | 2 +- .../configuration2}/XXEValidatorTest.java | 3 +- .../src/test/resources/local_xxe_file.xml | 0 .../src/test/resources/multiline_xxe_file.xml | 0 .../src/test/resources/no_xxe.xml | 0 .../src/test/resources/remote_xxe_file.xml | 0 .../test/resources/whitespace_xxe_file.xml | 0 .../pom.xml | 5 + .../WindowsEventLogRecordReader.java | 18 +- .../org/apache/nifi/xml/XMLRecordReader.java | 13 +- .../nifi/xml/inference/XmlRecordSource.java | 16 +- .../nifi-update-attribute-model/pom.xml | 2 +- .../attributes/serde/CriteriaSerDe.java | 12 +- .../nifi-registry-framework/pom.xml | 5 + .../IdentityProviderFactory.java | 6 +- .../AbstractPolicyBasedAuthorizer.java | 14 +- .../authorization/AuthorizerFactory.java | 13 +- .../StandardManagedAuthorizer.java | 14 +- .../file/FileAccessPolicyProvider.java | 13 +- .../file/FileUserGroupProvider.java | 13 +- .../nifi/registry/security/util/XmlUtils.java | 44 ---- .../alias/RegistryUrlAliasService.java | 12 +- .../service/extension/docs/XmlValidator.java | 26 +- .../nifi-registry-ranger-plugin/pom.xml | 7 + .../registry/ranger/RangerAuthorizer.java | 21 +- .../nifi-system-test-suite/pom.xml | 10 + .../JoinClusterWithDifferentFlow.java | 10 +- .../nifi-toolkit-flowanalyzer/pom.xml | 5 + .../flowanalyzer/FlowAnalyzerDriver.java | 8 +- 120 files changed, 2081 insertions(+), 1267 deletions(-) delete mode 100644 nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java delete mode 100644 nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy create mode 100644 nifi-commons/nifi-xml-processing/pom.xml create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingException.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingFeature.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/DocumentProvider.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProvider.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/InputSourceParser.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParser.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProvider.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProvider.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLEventReaderProvider.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLStreamReaderProvider.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/SchemaValidator.java create mode 100644 nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidator.java create mode 100644 nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/ResourceProvider.java create mode 100644 nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProviderTest.java create mode 100644 nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParserTest.java create mode 100644 nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProviderTest.java create mode 100644 nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProviderTest.java create mode 100644 nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidatorTest.java create mode 100644 nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype-entity.xml create mode 100644 nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype.xml create mode 100644 nifi-commons/nifi-xml-processing/src/test/resources/standard-document.xml create mode 100644 nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document-doctype-entity.xml create mode 100644 nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document.xml create mode 100644 nifi-commons/nifi-xml-processing/src/test/resources/standard-schema.xsd delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/groovy/org/apache/nifi/authorization/FlowParserTest.groovy delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-xxe.xml.gz create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/xml/DocumentTypeAllowedDocumentProvider.java rename {nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2}/SafeXMLConfiguration.java (97%) rename {nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2}/XXEValidator.java (99%) rename {nifi-commons/nifi-security-utils/src/test/java/org/apache/nifi/security => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/java/org/apache/nifi/lookup/configuration2}/XXEValidatorTest.java (98%) rename {nifi-commons/nifi-security-utils => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services}/src/test/resources/local_xxe_file.xml (100%) rename {nifi-commons/nifi-security-utils => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services}/src/test/resources/multiline_xxe_file.xml (100%) rename {nifi-commons/nifi-security-utils => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services}/src/test/resources/no_xxe.xml (100%) rename {nifi-commons/nifi-security-utils => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services}/src/test/resources/remote_xxe_file.xml (100%) rename {nifi-commons/nifi-security-utils => nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services}/src/test/resources/whitespace_xxe_file.xml (100%) delete mode 100644 nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/util/XmlUtils.java diff --git a/minifi/minifi-bootstrap/pom.xml b/minifi/minifi-bootstrap/pom.xml index cf2d83ad41..693d128072 100644 --- a/minifi/minifi-bootstrap/pom.xml +++ b/minifi/minifi-bootstrap/pom.xml @@ -56,6 +56,10 @@ limitations under the License. org.apache.nifi nifi-utils + + org.apache.nifi + nifi-xml-processing + org.apache.nifi nifi-properties diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 1d58e0a870..d3dc8eb382 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -46,15 +46,14 @@ import org.apache.nifi.minifi.commons.schema.common.Schema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; @@ -148,7 +147,7 @@ public final class ConfigTransformer { } } - protected static void writeFlowXmlFile(ConfigSchema configSchema, OutputStream outputStream) throws TransformerException, ConfigTransformerException, ConfigurationChangeException, IOException { + protected static void writeFlowXmlFile(ConfigSchema configSchema, OutputStream outputStream) throws TransformerException, ConfigTransformerException { final StreamResult streamResult = new StreamResult(outputStream); // configure the transformer and convert the DOM @@ -307,14 +306,12 @@ public final class ConfigTransformer { } } - protected static DOMSource createFlowXml(ConfigSchema configSchema) throws IOException, ConfigurationChangeException, ConfigTransformerException { + protected static DOMSource createFlowXml(ConfigSchema configSchema) throws ConfigTransformerException { try { // create a new, empty document - final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); - docFactory.setNamespaceAware(true); - - final DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); - final Document doc = docBuilder.newDocument(); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setNamespaceAware(true); + final Document doc = documentProvider.newDocument(); // populate document with controller state final Element rootNode = doc.createElement("flowController"); @@ -365,7 +362,7 @@ public final class ConfigTransformer { } return new DOMSource(doc); - } catch (final ParserConfigurationException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) { + } catch (final ProcessingException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) { throw new ConfigTransformerException(e); } catch (Exception e) { throw new ConfigTransformerException("Failed to parse the config YAML while writing the top level of the flow xml", e); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index bb01784f8b..7ca896c75b 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -34,6 +34,8 @@ import org.apache.nifi.minifi.commons.schema.common.StringUtil; import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.apache.nifi.util.StringUtils; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -42,9 +44,6 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; @@ -55,7 +54,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.StringBufferInputStream; import java.nio.charset.Charset; import java.util.Arrays; import java.util.Collections; @@ -80,15 +78,14 @@ public class ConfigTransformerTest { Arrays.asList("processor", "inputPort", "outputPort", "funnel", "processGroup", "remoteProcessGroup", "connection")); private XPathFactory xPathFactory; private Element config; - private DocumentBuilder documentBuilder; @Rule final public TemporaryFolder tempOutputFolder = new TemporaryFolder(); @Before - public void setup() throws ParserConfigurationException { - documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - final Document document = documentBuilder.newDocument(); + public void setup() { + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.newDocument(); config = document.createElement("config"); xPathFactory = XPathFactory.newInstance(); } @@ -484,11 +481,11 @@ public class ConfigTransformerTest { assertTrue(flowXml.exists()); assertTrue(flowXml.canRead()); - String flow = loadFlowXML(new FileInputStream(flowXml)); - - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - Document xml = db.parse(new StringBufferInputStream(flow)); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document xml; + try (final InputStream inputStream = new GZIPInputStream(new FileInputStream(flowXml))) { + xml = documentProvider.parse(inputStream); + } XPath xPath = XPathFactory.newInstance().newXPath(); String result = xPath.evaluate("/flowController/rootGroup/processor/property[name = \"SSL Context Service\"]/value/text()", xml); @@ -504,7 +501,8 @@ public class ConfigTransformerTest { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ConfigTransformer.writeFlowXmlFile(configSchema, outputStream); - Document document = documentBuilder.parse(new ByteArrayInputStream(outputStream.toByteArray())); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + Document document = documentProvider.parse(new ByteArrayInputStream(outputStream.toByteArray())); testProcessGroup((Element) xPathFactory.newXPath().evaluate("flowController/rootGroup", document, XPathConstants.NODE), configSchema.getProcessGroupSchema()); testReportingTasks((Element) xPathFactory.newXPath().evaluate("flowController/reportingTasks", document, XPathConstants.NODE), configSchema.getReportingTasksSchema()); @@ -786,17 +784,4 @@ public class ConfigTransformerTest { } return bootstrapProperties; } - - public static String loadFlowXML(InputStream compressedData) throws IOException { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - GZIPInputStream gzipInputStream = new GZIPInputStream(compressedData); - - byte[] buffer = new byte[1024]; - int len; - while ((len = gzipInputStream.read(buffer)) != -1) { - byteArrayOutputStream.write(buffer, 0, len); - } - - return byteArrayOutputStream.toString(); - } } diff --git a/minifi/pom.xml b/minifi/pom.xml index 06b71f58e2..6f9d7d7f10 100644 --- a/minifi/pom.xml +++ b/minifi/pom.xml @@ -385,6 +385,11 @@ limitations under the License. nifi-security-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.nifi nifi-logging-utils diff --git a/nifi-bootstrap/pom.xml b/nifi-bootstrap/pom.xml index 6d592fefa5..cb1214f7f3 100644 --- a/nifi-bootstrap/pom.xml +++ b/nifi-bootstrap/pom.xml @@ -45,6 +45,11 @@ language governing permissions and limitations under the License. --> nifi-security-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.nifi nifi-flow-encryptor diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java index e7d524b587..97582fe8c9 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java @@ -31,18 +31,14 @@ import org.apache.nifi.components.resource.StandardResourceContext; import org.apache.nifi.components.resource.StandardResourceReferenceFactory; import org.apache.nifi.parameter.ParameterLookup; import org.apache.nifi.registry.VariableRegistry; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -91,27 +87,6 @@ public class NotificationServiceManager { this.maxAttempts = maxAttempts; } - private static DocumentBuilder createSafeDocumentBuilder() throws ParserConfigurationException { - final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); - docFactory.setNamespaceAware(false); - - // These features are used to disable processing external entities in the DocumentBuilderFactory to protect against XXE attacks - final String DISALLOW_DOCTYPES = "http://apache.org/xml/features/disallow-doctype-decl"; - final String ALLOW_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; - final String ALLOW_EXTERNAL_PARAM_ENTITIES = "http://xml.org/sax/features/external-parameter-entities"; - final String ALLOW_EXTERNAL_DTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd"; - - // Disable DTDs and external entities to protect against XXE - docFactory.setAttribute(DISALLOW_DOCTYPES, true); - docFactory.setAttribute(ALLOW_EXTERNAL_DTD, false); - docFactory.setAttribute(ALLOW_EXTERNAL_GENERAL_ENTITIES, false); - docFactory.setAttribute(ALLOW_EXTERNAL_PARAM_ENTITIES, false); - docFactory.setXIncludeAware(false); - docFactory.setExpandEntityReferences(false); - - return docFactory.newDocumentBuilder(); - } - /** * Loads the Notification Services from the given XML configuration file. * @@ -143,17 +118,14 @@ public class NotificationServiceManager { * * @param servicesFile the XML file to load services from. * @throws IOException if unable to read from the given file - * @throws ParserConfigurationException if unable to parse the given file as XML properly - * @throws SAXException if unable to parse the given file properly */ - public void loadNotificationServices(final File servicesFile) throws IOException, ParserConfigurationException, SAXException { - final DocumentBuilder docBuilder = createSafeDocumentBuilder(); - + public void loadNotificationServices(final File servicesFile) throws IOException { final Map serviceMap = new HashMap<>(); try (final InputStream fis = new FileInputStream(servicesFile); final InputStream in = new BufferedInputStream(fis)) { - final Document doc = docBuilder.parse(new InputSource(in)); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + final Document doc = documentProvider.parse(in); final List serviceElements = getChildElementsByTagName(doc.getDocumentElement(), "service"); logger.debug("Found {} service elements", serviceElements.size()); diff --git a/nifi-commons/nifi-security-utils/pom.xml b/nifi-commons/nifi-security-utils/pom.xml index 6ecfb43f7f..73bf668db3 100644 --- a/nifi-commons/nifi-security-utils/pom.xml +++ b/nifi-commons/nifi-security-utils/pom.xml @@ -42,11 +42,6 @@ nifi-security-utils-api 1.17.0-SNAPSHOT - - org.apache.nifi - nifi-security-kms - 1.17.0-SNAPSHOT - org.apache.commons commons-lang3 @@ -73,62 +68,6 @@ bcrypt 0.9.0 - - org.apache.commons - commons-configuration2 - 2.7 - - - commons-logging - commons-logging - - - - - org.slf4j - jcl-over-slf4j - - - - - - org.apache.rat - apache-rat-plugin - - - src/test/resources/xxe_template.xml - - - - - - - - - - jigsaw - - (1.8,) - - - - jakarta.xml.bind - jakarta.xml.bind-api - test - - - org.glassfish.jaxb - jaxb-runtime - test - - - com.sun.activation - javax.activation - - - - diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java deleted file mode 100644 index 0066806b7c..0000000000 --- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.security.xml; - -import java.io.InputStream; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; -import javax.xml.transform.stream.StreamSource; -import javax.xml.validation.Schema; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; - -public class XmlUtils { - - // These features are used to disable processing external entities in the DocumentBuilderFactory to protect against XXE attacks - private static final String DISALLOW_DOCTYPES = "http://apache.org/xml/features/disallow-doctype-decl"; - private static final String ALLOW_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; - private static final String ALLOW_EXTERNAL_PARAM_ENTITIES = "http://xml.org/sax/features/external-parameter-entities"; - private static final String ALLOW_EXTERNAL_DTD = "http://apache.org/xml/features/nonvalidating/load-external-dtd"; - - public static XMLStreamReader createSafeReader(InputStream inputStream) throws XMLStreamException { - if (inputStream == null) { - throw new IllegalArgumentException("The provided input stream cannot be null"); - } - return createSafeReader(new StreamSource(inputStream)); - } - - public static XMLStreamReader createSafeReader(StreamSource source) throws XMLStreamException { - if (source == null) { - throw new IllegalArgumentException("The provided source cannot be null"); - } - - XMLInputFactory xif = XMLInputFactory.newFactory(); - xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); - return xif.createXMLStreamReader(source); - } - - public static XMLReader createSafeSaxReader(SAXParserFactory saxParserFactory, ContentHandler contentHandler) throws SAXException, ParserConfigurationException { - if (saxParserFactory == null) { - throw new IllegalArgumentException("The provided SAX parser factory cannot be null"); - } - - if (contentHandler == null) { - throw new IllegalArgumentException("The provided SAX content handler cannot be null"); - } - - SAXParser saxParser = saxParserFactory.newSAXParser(); - XMLReader xmlReader = saxParser.getXMLReader(); - xmlReader.setFeature(DISALLOW_DOCTYPES, true); - xmlReader.setFeature(ALLOW_EXTERNAL_GENERAL_ENTITIES, false); - xmlReader.setFeature(ALLOW_EXTERNAL_PARAM_ENTITIES, false); - xmlReader.setContentHandler(contentHandler); - - return xmlReader; - } - - public static DocumentBuilder createSafeDocumentBuilder(Schema schema, boolean isNamespaceAware) throws ParserConfigurationException { - final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); - docFactory.setSchema(schema); - docFactory.setNamespaceAware(isNamespaceAware); - - // Disable DTDs and external entities to protect against XXE - docFactory.setAttribute(DISALLOW_DOCTYPES, true); - docFactory.setAttribute(ALLOW_EXTERNAL_DTD, false); - docFactory.setAttribute(ALLOW_EXTERNAL_GENERAL_ENTITIES, false); - docFactory.setAttribute(ALLOW_EXTERNAL_PARAM_ENTITIES, false); - docFactory.setXIncludeAware(false); - docFactory.setExpandEntityReferences(false); - - return docFactory.newDocumentBuilder(); - } - - public static DocumentBuilder createSafeDocumentBuilder(Schema schema) throws ParserConfigurationException { - return createSafeDocumentBuilder(schema, true); - } - - public static DocumentBuilder createSafeDocumentBuilder(boolean isNamespaceAware) throws ParserConfigurationException { - return createSafeDocumentBuilder(null, isNamespaceAware); - } -} diff --git a/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy deleted file mode 100644 index fb12bfec00..0000000000 --- a/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.security.xml - -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import org.xml.sax.SAXParseException - -import javax.xml.bind.JAXBContext -import javax.xml.bind.UnmarshalException -import javax.xml.bind.Unmarshaller -import javax.xml.bind.annotation.XmlAccessType -import javax.xml.bind.annotation.XmlAccessorType -import javax.xml.bind.annotation.XmlAttribute -import javax.xml.bind.annotation.XmlRootElement -import javax.xml.parsers.DocumentBuilder -import javax.xml.stream.XMLStreamReader - -import static groovy.test.GroovyAssert.shouldFail - -class XmlUtilsTest { - private static final Logger logger = LoggerFactory.getLogger(XmlUtilsTest.class) - - @BeforeAll - static void setUpOnce() throws Exception { - logger.metaClass.methodMissing = { String name, args -> - logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") - } - } - - @Test - void testShouldHandleXXEInUnmarshal() { - // Arrange - final String XXE_TEMPLATE_FILEPATH = "src/test/resources/local_xxe_file.xml" - InputStream templateStream = new File(XXE_TEMPLATE_FILEPATH).newInputStream() - - JAXBContext context = JAXBContext.newInstance(XmlObject.class) - - // Act - def msg = shouldFail(UnmarshalException) { - Unmarshaller unmarshaller = context.createUnmarshaller() - XMLStreamReader xsr = XmlUtils.createSafeReader(templateStream) - def parsed = unmarshaller.unmarshal(xsr, XmlObject.class) - logger.info("Unmarshalled ${parsed.toString()}") - } - - // Assert - logger.expected(msg) - assert msg =~ "XMLStreamException: ParseError " - } - - @Test - void testShouldHandleXXEInDocumentBuilder() { - // Arrange - final String XXE_TEMPLATE_FILEPATH = "src/test/resources/local_xxe_file.xml" - DocumentBuilder documentBuilder = XmlUtils.createSafeDocumentBuilder(null) - - // Act - def msg = shouldFail(SAXParseException) { - def parsedFlow = documentBuilder.parse(new File(XXE_TEMPLATE_FILEPATH)) - logger.info("Parsed ${parsedFlow.toString()}") - } - - // Assert - logger.expected(msg) - assert msg =~ "SAXParseException.*DOCTYPE" - } -} - -@XmlAccessorType(XmlAccessType.NONE) -@XmlRootElement(name = "object") -class XmlObject { - @XmlAttribute - String name - - @XmlAttribute - String description - - @XmlAttribute - String groupId - - @XmlAttribute - String timestamp -} diff --git a/nifi-commons/nifi-single-user-utils/pom.xml b/nifi-commons/nifi-single-user-utils/pom.xml index 4584673a7c..7a9bee774d 100644 --- a/nifi-commons/nifi-single-user-utils/pom.xml +++ b/nifi-commons/nifi-single-user-utils/pom.xml @@ -22,6 +22,11 @@ nifi-single-user-utils + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + at.favre.lib bcrypt diff --git a/nifi-commons/nifi-single-user-utils/src/main/java/org/apache/nifi/authentication/single/user/writer/StandardLoginCredentialsWriter.java b/nifi-commons/nifi-single-user-utils/src/main/java/org/apache/nifi/authentication/single/user/writer/StandardLoginCredentialsWriter.java index 520b330d2b..c834252fdf 100644 --- a/nifi-commons/nifi-single-user-utils/src/main/java/org/apache/nifi/authentication/single/user/writer/StandardLoginCredentialsWriter.java +++ b/nifi-commons/nifi-single-user-utils/src/main/java/org/apache/nifi/authentication/single/user/writer/StandardLoginCredentialsWriter.java @@ -17,11 +17,12 @@ package org.apache.nifi.authentication.single.user.writer; import org.apache.nifi.authentication.single.user.SingleUserCredentials; +import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider; import javax.xml.stream.XMLEventFactory; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLEventWriter; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.Attribute; @@ -29,6 +30,7 @@ import javax.xml.stream.events.Characters; import javax.xml.stream.events.EndElement; import javax.xml.stream.events.StartElement; import javax.xml.stream.events.XMLEvent; +import javax.xml.transform.stream.StreamSource; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; @@ -199,10 +201,8 @@ public class StandardLoginCredentialsWriter implements LoginCredentialsWriter { return outputFactory.createXMLEventWriter(outputStream); } - private XMLEventReader getProvidersReader(final InputStream inputStream) throws XMLStreamException { - final XMLInputFactory inputFactory = XMLInputFactory.newFactory(); - inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - return inputFactory.createXMLEventReader(inputStream); + private XMLEventReader getProvidersReader(final InputStream inputStream) { + final XMLEventReaderProvider readerProvider = new StandardXMLEventReaderProvider(); + return readerProvider.getEventReader(new StreamSource(inputStream)); } } diff --git a/nifi-commons/nifi-site-to-site-client/pom.xml b/nifi-commons/nifi-site-to-site-client/pom.xml index 5a213c6d7f..bf1fc68458 100644 --- a/nifi-commons/nifi-site-to-site-client/pom.xml +++ b/nifi-commons/nifi-site-to-site-client/pom.xml @@ -51,6 +51,11 @@ nifi-security-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.commons commons-lang3 diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java index 5a0264e4d4..6963701cf2 100644 --- a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java +++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java @@ -25,9 +25,12 @@ import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import org.apache.nifi.security.xml.XmlUtils; +import javax.xml.transform.stream.StreamSource; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; @XmlRootElement public class ClusterNodeInformation { @@ -65,9 +68,10 @@ public class ClusterNodeInformation { public static ClusterNodeInformation unmarshal(final InputStream is) throws JAXBException { try { final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - final XMLStreamReader xsr = XmlUtils.createSafeReader(is); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(is)); return (ClusterNodeInformation) unmarshaller.unmarshal(xsr); - } catch (XMLStreamException e) { + } catch (final ProcessingException e) { throw new JAXBException("Error unmarshalling the cluster node information", e); } } diff --git a/nifi-commons/nifi-xml-processing/pom.xml b/nifi-commons/nifi-xml-processing/pom.xml new file mode 100644 index 0000000000..6eb9ba88c0 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-commons + 1.17.0-SNAPSHOT + + nifi-xml-processing + 1.17.0-SNAPSHOT + jar + + + + com.github.spotbugs + spotbugs-maven-plugin + 4.6.0.0 + + + package + + check + + + + + Max + low + true + + + com.h3xstream.findsecbugs + findsecbugs-plugin + 1.12.0 + + + + + + + diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingException.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingException.java new file mode 100644 index 0000000000..d0d186d710 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing; + +/** + * General Exception for XML processing problems + */ +public class ProcessingException extends RuntimeException { + /** + * Processing Exception with message and cause for tracing + * + * @param message Error Message + * @param cause Throwable cause for tracing + */ + public ProcessingException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingFeature.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingFeature.java new file mode 100644 index 0000000000..ee4946cd64 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/ProcessingFeature.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing; + +import javax.xml.XMLConstants; + +/** + * XML Processing Features + */ +public enum ProcessingFeature { + /** Secure Processing */ + SECURE_PROCESSING(XMLConstants.FEATURE_SECURE_PROCESSING, true), + + /** SAX Namespaces */ + SAX_NAMESPACES("http://xml.org/sax/features/namespaces", true), + + /** SAX Namespace Prefixes */ + SAX_NAMESPACE_PREFIXES("http://xml.org/sax/features/namespace-prefixes", true), + + /** Disallow Document Type Declaration */ + DISALLOW_DOCTYPE_DECL("http://apache.org/xml/features/disallow-doctype-decl", true); + + private final String feature; + + private final boolean enabled; + + ProcessingFeature(final String feature, final boolean enabled) { + this.feature = feature; + this.enabled = enabled; + } + + public String getFeature() { + return feature; + } + + public boolean isEnabled() { + return enabled; + } +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/DocumentProvider.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/DocumentProvider.java new file mode 100644 index 0000000000..20c9ba5369 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/DocumentProvider.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.parsers; + +import org.w3c.dom.Document; + +import java.io.InputStream; + +/** + * Provider for instances of Document Object Model + */ +public interface DocumentProvider { + /** + * Create new Document + * + * @return Document + */ + Document newDocument(); + + /** + * Parse InputStream to Document + * + * @param inputStream InputStream to be parsed + * @return Document parsed from stream + */ + Document parse(InputStream inputStream); +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProvider.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProvider.java new file mode 100644 index 0000000000..e457fdc5a5 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProvider.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.parsers; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.ProcessingFeature; +import org.w3c.dom.Document; +import org.xml.sax.ErrorHandler; +import org.xml.sax.SAXException; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.validation.Schema; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; + +/** + * Standard implementation of Document Provider with secure processing enabled + */ +public class StandardDocumentProvider implements DocumentProvider { + private boolean namespaceAware; + + private Schema schema; + + private ErrorHandler errorHandler; + + /** + * Set Error Handler + * + * @param errorHandler Error Handler + */ + public void setErrorHandler(final ErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } + + /** + * Set Namespace Aware status on DocumentBuilderFactory + * + * @param namespaceAware Namespace Awareness + */ + public void setNamespaceAware(final boolean namespaceAware) { + this.namespaceAware = namespaceAware; + } + + /** + * Set Namespace Aware status on DocumentBuilderFactory + * + * @param schema Schema for validation or null to disable validation + */ + public void setSchema(final Schema schema) { + this.schema = schema; + } + + @Override + public Document newDocument() { + final DocumentBuilderFactory documentBuilderFactory = getDocumentBuilderFactory(); + + try { + documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled()); + documentBuilderFactory.setFeature(ProcessingFeature.DISALLOW_DOCTYPE_DECL.getFeature(), ProcessingFeature.DISALLOW_DOCTYPE_DECL.isEnabled()); + + final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); + documentBuilder.setErrorHandler(errorHandler); + + return documentBuilder.newDocument(); + } catch (final ParserConfigurationException e) { + throw new ProcessingException("Configuration failed", e); + } + } + + /** + * Build and return DocumentBuilder + * + * @return DocumentBuilder configured using provided properties + */ + @Override + public Document parse(final InputStream inputStream) { + Objects.requireNonNull(inputStream, "InputStream required"); + final DocumentBuilderFactory documentBuilderFactory = getDocumentBuilderFactory(); + + try { + documentBuilderFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled()); + documentBuilderFactory.setFeature(ProcessingFeature.DISALLOW_DOCTYPE_DECL.getFeature(), isDisallowDocumentTypeDeclaration()); + + final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); + documentBuilder.setErrorHandler(errorHandler); + + return documentBuilder.parse(inputStream); + } catch (final ParserConfigurationException|SAXException|IOException e) { + throw new ProcessingException("Parsing failed", e); + } + } + + protected boolean isDisallowDocumentTypeDeclaration() { + return ProcessingFeature.DISALLOW_DOCTYPE_DECL.isEnabled(); + } + + private DocumentBuilderFactory getDocumentBuilderFactory() { + final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + + documentBuilderFactory.setSchema(schema); + documentBuilderFactory.setNamespaceAware(namespaceAware); + + documentBuilderFactory.setXIncludeAware(false); + documentBuilderFactory.setExpandEntityReferences(false); + + return documentBuilderFactory; + } +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/InputSourceParser.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/InputSourceParser.java new file mode 100644 index 0000000000..bb50dc7ced --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/InputSourceParser.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.sax; + +import org.xml.sax.ContentHandler; +import org.xml.sax.InputSource; + +/** + * SAX Input Source Parser + */ +public interface InputSourceParser { + /** + * Parse Input Source using Content Handler + * + * @param inputSource Input Source to be parsed + * @param contentHandler Content Handler used during parsing + */ + void parse(InputSource inputSource, ContentHandler contentHandler); +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParser.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParser.java new file mode 100644 index 0000000000..14f7b31ed6 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParser.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.sax; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.ProcessingFeature; +import org.xml.sax.ContentHandler; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; + +import javax.xml.XMLConstants; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import java.io.IOException; +import java.util.Objects; + +/** + * Standard implementation of Input Source Parser with secure processing enabled + */ +public class StandardInputSourceParser implements InputSourceParser { + private boolean namespaceAware; + + /** + * Set Namespace Aware status on SAXParserFactory + * + * @param namespaceAware Namespace Aware status + */ + public void setNamespaceAware(final boolean namespaceAware) { + this.namespaceAware = namespaceAware; + } + + /** + * Parse Input Source using Content Handler + * + * @param inputSource Input Source to be parsed + * @param contentHandler Content Handler used during parsing + */ + @Override + public void parse(final InputSource inputSource, final ContentHandler contentHandler) { + Objects.requireNonNull(inputSource, "InputSource required"); + Objects.requireNonNull(contentHandler, "ContentHandler required"); + + try { + parseInputSource(inputSource, contentHandler); + } catch (final ParserConfigurationException|SAXException e) { + throw new ProcessingException("Parsing failed", e); + } + } + + private void parseInputSource(final InputSource inputSource, final ContentHandler contentHandler) throws ParserConfigurationException, SAXException { + final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setNamespaceAware(namespaceAware); + saxParserFactory.setXIncludeAware(false); + + saxParserFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled()); + saxParserFactory.setFeature(ProcessingFeature.DISALLOW_DOCTYPE_DECL.getFeature(), ProcessingFeature.DISALLOW_DOCTYPE_DECL.isEnabled()); + + if (namespaceAware) { + saxParserFactory.setFeature(ProcessingFeature.SAX_NAMESPACES.getFeature(), ProcessingFeature.SAX_NAMESPACES.isEnabled()); + saxParserFactory.setFeature(ProcessingFeature.SAX_NAMESPACE_PREFIXES.getFeature(), ProcessingFeature.SAX_NAMESPACE_PREFIXES.isEnabled()); + } + + final SAXParser parser = saxParserFactory.newSAXParser(); + + final XMLReader reader = parser.getXMLReader(); + reader.setContentHandler(contentHandler); + + try { + reader.parse(inputSource); + } catch (final IOException e) { + throw new ProcessingException("Parsing failed", e); + } + } +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProvider.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProvider.java new file mode 100644 index 0000000000..dbfcb282fe --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.stream; + +import org.apache.nifi.xml.processing.ProcessingException; + +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.transform.stream.StreamSource; +import java.util.Objects; + +/** + * Standard implementation of XMLStreamReader provider with secure processing enabled + */ +public class StandardXMLEventReaderProvider implements XMLEventReaderProvider { + /** + * Get XML Event Reader + * + * @param streamSource Stream Source for Reader + * @return Configured XML Event Reader + */ + @Override + public XMLEventReader getEventReader(final StreamSource streamSource) { + Objects.requireNonNull(streamSource, "StreamSource required"); + + final XMLInputFactory inputFactory = XMLInputFactory.newFactory(); + inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); + + try { + return inputFactory.createXMLEventReader(streamSource); + } catch (final XMLStreamException e) { + throw new ProcessingException("Reader creation failed", e); + } + } +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProvider.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProvider.java new file mode 100644 index 0000000000..fdefd693a2 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.stream; + +import org.apache.nifi.xml.processing.ProcessingException; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; +import java.util.Objects; + +/** + * Standard implementation of XMLStreamReader provider with secure processing enabled + */ +public class StandardXMLStreamReaderProvider implements XMLStreamReaderProvider { + /** + * Get XML Stream Reader with external entities disabled + * + * @param streamSource Stream Source for Reader + * @return Configured XML Stream Reader + */ + @Override + public XMLStreamReader getStreamReader(final StreamSource streamSource) { + Objects.requireNonNull(streamSource, "StreamSource required"); + + final XMLInputFactory inputFactory = XMLInputFactory.newFactory(); + inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); + + try { + return inputFactory.createXMLStreamReader(streamSource); + } catch (final XMLStreamException e) { + throw new ProcessingException("Reader creation failed", e); + } + } +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLEventReaderProvider.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLEventReaderProvider.java new file mode 100644 index 0000000000..a67fe66c70 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLEventReaderProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.stream; + +import javax.xml.stream.XMLEventReader; +import javax.xml.transform.stream.StreamSource; + +/** + * Provider for instances of XMLEventReader + */ +public interface XMLEventReaderProvider { + /** + * Get XML Event Reader + * + * @param streamSource Stream Source for Reader + * @return Configured XML Event Reader + */ + XMLEventReader getEventReader(StreamSource streamSource); +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLStreamReaderProvider.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLStreamReaderProvider.java new file mode 100644 index 0000000000..fe1deac028 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/stream/XMLStreamReaderProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.stream; + +import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; + +/** + * Provider for instances of XMLStreamReader + */ +public interface XMLStreamReaderProvider { + /** + * Get XML Stream Reader + * + * @param streamSource Stream Source for Reader + * @return Configured XML Stream Reader + */ + XMLStreamReader getStreamReader(StreamSource streamSource); +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/SchemaValidator.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/SchemaValidator.java new file mode 100644 index 0000000000..a8800debd6 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/SchemaValidator.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.validation; + +import javax.xml.transform.Source; +import javax.xml.validation.Schema; + +/** + * XML Schema Validator + */ +public interface SchemaValidator { + /** + * Validate Source using Schema + * + * @param schema Schema source for Validator + * @param source Source to be validated + */ + void validate(Schema schema, Source source); +} diff --git a/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidator.java b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidator.java new file mode 100644 index 0000000000..940d7f8cea --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/main/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.validation; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.ProcessingFeature; +import org.xml.sax.SAXException; + +import javax.xml.XMLConstants; +import javax.xml.transform.Source; +import javax.xml.validation.Schema; +import javax.xml.validation.Validator; +import java.io.IOException; +import java.util.Objects; + +/** + * Standard implementation of XML Schema Validator with secure processing enabled + */ +public class StandardSchemaValidator implements SchemaValidator { + /** + * Validate Source using Schema + * + * @param schema Schema source for Validator + * @param source Source to be validated + */ + @Override + public void validate(final Schema schema, final Source source) { + Objects.requireNonNull(schema, "Schema required"); + Objects.requireNonNull(source, "Source required"); + + final Validator validator = schema.newValidator(); + + try { + validator.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, ProcessingFeature.SECURE_PROCESSING.isEnabled()); + } catch (final SAXException e) { + throw new ProcessingException("Validator configuration failed", e); + } + + try { + validator.validate(source); + } catch (final SAXException|IOException e) { + throw new ProcessingException("Validation failed", e); + } + } +} diff --git a/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/ResourceProvider.java b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/ResourceProvider.java new file mode 100644 index 0000000000..223165c926 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/ResourceProvider.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing; + +import java.io.InputStream; + +public class ResourceProvider { + private static final String STANDARD_DOCUMENT_DOCTYPE_ENTITY = "/standard-document-doctype-entity.xml"; + + private static final String STANDARD_DOCUMENT_DOCTYPE = "/standard-document-doctype.xml"; + + private static final String STANDARD_DOCUMENT = "/standard-document.xml"; + + private static final String STANDARD_NAMESPACE_DOCUMENT = "/standard-namespace-document.xml"; + + private static final String STANDARD_NAMESPACE_DOCUMENT_DOCTYPE_ENTITY = "/standard-namespace-document-doctype-entity.xml"; + + private static final String STANDARD_SCHEMA = "/standard-schema.xsd"; + + public static InputStream getStandardDocument() { + return getResource(STANDARD_DOCUMENT); + } + + public static InputStream getStandardDocumentDocTypeEntity() { + return getResource(STANDARD_DOCUMENT_DOCTYPE_ENTITY); + } + + public static InputStream getStandardDocumentDocType() { + return getResource(STANDARD_DOCUMENT_DOCTYPE); + } + + public static InputStream getStandardNamespaceDocument() { + return getResource(STANDARD_NAMESPACE_DOCUMENT); + } + + public static InputStream getStandardNamespaceDocumentDocTypeEntity() { + return getResource(STANDARD_NAMESPACE_DOCUMENT_DOCTYPE_ENTITY); + } + + public static InputStream getStandardSchema() { + return getResource(STANDARD_SCHEMA); + } + + private static InputStream getResource(final String path) { + final InputStream resource = ResourceProvider.class.getResourceAsStream(path); + if (resource == null) { + throw new IllegalStateException(String.format("Resource [%s] not found", path)); + } + return resource; + } +} diff --git a/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProviderTest.java b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProviderTest.java new file mode 100644 index 0000000000..15d32f9a1e --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/parsers/StandardDocumentProviderTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.parsers; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.ResourceProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.w3c.dom.Document; +import org.xml.sax.ErrorHandler; +import org.xml.sax.SAXParseException; + +import javax.xml.validation.Schema; +import javax.xml.validation.ValidatorHandler; + +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class StandardDocumentProviderTest { + @Mock + Schema schema; + + @Mock + ValidatorHandler validatorHandler; + + @Mock + ErrorHandler errorHandler; + + @Test + void testNewDocument() { + final StandardDocumentProvider provider = new StandardDocumentProvider(); + + final Document document = provider.newDocument(); + + assertNotNull(document); + } + + @Test + void testParseStandard() throws IOException { + final StandardDocumentProvider provider = new StandardDocumentProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocument()) { + final Document document = provider.parse(inputStream); + assertNotNull(document); + } + } + + @Test + void testParseDocumentTypeDeclarationException() throws IOException { + final StandardDocumentProvider provider = new StandardDocumentProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) { + assertParsingException(inputStream, provider); + } + } + + @Test + void testParseExternalEntityException() throws IOException { + final StandardDocumentProvider provider = new StandardDocumentProvider(); + + assertParsingException(provider); + } + + @Test + void testParseNamespaceAwareSchemaConfiguredExternalEntityException() throws IOException { + when(schema.newValidatorHandler()).thenReturn(validatorHandler); + + final StandardDocumentProvider provider = new StandardDocumentProvider(); + provider.setNamespaceAware(true); + provider.setSchema(schema); + provider.setErrorHandler(errorHandler); + + assertParsingException(provider); + } + + private void assertParsingException(final StandardDocumentProvider provider) throws IOException { + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) { + assertParsingException(inputStream, provider); + } + } + + private void assertParsingException(final InputStream inputStream, final StandardDocumentProvider provider) { + final ProcessingException processingException = assertThrows(ProcessingException.class, () -> provider.parse(inputStream)); + assertInstanceOf(SAXParseException.class, processingException.getCause()); + } +} diff --git a/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParserTest.java b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParserTest.java new file mode 100644 index 0000000000..74d8277fcf --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/sax/StandardInputSourceParserTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.sax; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.ResourceProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.xml.sax.ContentHandler; +import org.xml.sax.InputSource; +import org.xml.sax.SAXParseException; +import org.xml.sax.helpers.DefaultHandler; + +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@ExtendWith(MockitoExtension.class) +public class StandardInputSourceParserTest { + @Mock + ContentHandler contentHandler; + + @Test + void testParseStandard() throws IOException { + final StandardInputSourceParser parser = new StandardInputSourceParser(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocument()) { + parser.parse(new InputSource(inputStream), contentHandler); + } + } + + @Test + void testParseDocumentTypeDeclarationException() throws IOException { + final StandardInputSourceParser parser = new StandardInputSourceParser(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) { + assertParsingException(inputStream, parser); + } + } + + @Test + void testParseExternalEntityException() throws IOException { + final StandardInputSourceParser parser = new StandardInputSourceParser(); + + assertParsingException(parser); + } + + @Test + void testParseNamespaceAwareExternalEntityException() throws IOException { + final StandardInputSourceParser parser = new StandardInputSourceParser(); + + parser.setNamespaceAware(true); + + assertParsingException(parser); + } + + private void assertParsingException(final StandardInputSourceParser parser) throws IOException { + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) { + assertParsingException(inputStream, parser); + } + } + + private void assertParsingException(final InputStream inputStream, final StandardInputSourceParser parser) { + final ProcessingException processingException = assertThrows(ProcessingException.class, () -> parser.parse(new InputSource(inputStream), new DefaultHandler())); + assertInstanceOf(SAXParseException.class, processingException.getCause()); + } +} diff --git a/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProviderTest.java b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProviderTest.java new file mode 100644 index 0000000000..32753a0a9a --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLEventReaderProviderTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.stream; + +import org.apache.nifi.xml.processing.ResourceProvider; +import org.junit.jupiter.api.Test; + +import javax.xml.stream.XMLEventReader; +import javax.xml.stream.XMLStreamException; +import javax.xml.transform.stream.StreamSource; +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class StandardXMLEventReaderProviderTest { + @Test + void testGetEventReaderStandard() throws IOException, XMLStreamException { + final StandardXMLEventReaderProvider provider = new StandardXMLEventReaderProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocument()) { + final XMLEventReader reader = provider.getEventReader(new StreamSource(inputStream)); + processReader(reader); + } + } + + @Test + void testGetEventReaderStandardDocumentTypeDeclaration() throws IOException { + final StandardXMLEventReaderProvider provider = new StandardXMLEventReaderProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) { + final XMLEventReader reader = provider.getEventReader(new StreamSource(inputStream)); + assertDoesNotThrow(() -> processReader(reader)); + } + } + + @Test + void testGetEventReaderStandardExternalEntityException() throws IOException { + final StandardXMLEventReaderProvider provider = new StandardXMLEventReaderProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) { + final XMLEventReader reader = provider.getEventReader(new StreamSource(inputStream)); + assertThrows(XMLStreamException.class, () -> processReader(reader)); + } + } + + private void processReader(final XMLEventReader reader) throws XMLStreamException { + while (reader.hasNext()) { + reader.nextEvent(); + } + } +} diff --git a/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProviderTest.java b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProviderTest.java new file mode 100644 index 0000000000..31ffb76522 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/stream/StandardXMLStreamReaderProviderTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.stream; + +import org.apache.nifi.xml.processing.ResourceProvider; +import org.junit.jupiter.api.Test; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class StandardXMLStreamReaderProviderTest { + @Test + void testGetStreamReaderStandard() throws IOException, XMLStreamException { + final StandardXMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocument()) { + final XMLStreamReader reader = provider.getStreamReader(new StreamSource(inputStream)); + processReader(reader); + } + } + + @Test + void testGetStreamReaderStandardDocumentTypeDeclaration() throws IOException { + final StandardXMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocType()) { + final XMLStreamReader reader = provider.getStreamReader(new StreamSource(inputStream)); + assertDoesNotThrow(() -> processReader(reader)); + } + } + + @Test + void testGetStreamReaderStandardExternalEntityException() throws IOException { + final StandardXMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + + try (final InputStream inputStream = ResourceProvider.getStandardDocumentDocTypeEntity()) { + final XMLStreamReader reader = provider.getStreamReader(new StreamSource(inputStream)); + assertThrows(XMLStreamException.class, () -> processReader(reader)); + } + } + + private void processReader(final XMLStreamReader reader) throws XMLStreamException { + while (reader.hasNext()) { + reader.next(); + } + } +} diff --git a/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidatorTest.java b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidatorTest.java new file mode 100644 index 0000000000..420bf076a7 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/java/org/apache/nifi/xml/processing/validation/StandardSchemaValidatorTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.xml.processing.validation; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.ResourceProvider; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; + +import java.io.IOException; +import java.io.InputStream; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class StandardSchemaValidatorTest { + private static final String SCHEMA_LANGUAGE = "http://www.w3.org/2001/XMLSchema"; + + @Test + void testValidate() throws SAXException, IOException { + final SchemaFactory schemaFactory = SchemaFactory.newInstance(SCHEMA_LANGUAGE); + final Schema schema; + try (final InputStream inputStream = ResourceProvider.getStandardSchema()) { + schema = schemaFactory.newSchema(new StreamSource(inputStream)); + } + + final StandardSchemaValidator validator = new StandardSchemaValidator(); + + try (final InputStream inputStream = ResourceProvider.getStandardNamespaceDocument()) { + validator.validate(schema, new StreamSource(inputStream)); + } + } + + @Test + void testValidateExternalEntityException() throws SAXException, IOException { + final SchemaFactory schemaFactory = SchemaFactory.newInstance(SCHEMA_LANGUAGE); + final Schema schema; + try (final InputStream inputStream = ResourceProvider.getStandardSchema()) { + schema = schemaFactory.newSchema(new StreamSource(inputStream)); + } + + final StandardSchemaValidator validator = new StandardSchemaValidator(); + + try (final InputStream inputStream = ResourceProvider.getStandardNamespaceDocumentDocTypeEntity()) { + final ProcessingException exception = assertThrows(ProcessingException.class, () -> validator.validate(schema, new StreamSource(inputStream))); + final Throwable cause = exception.getCause(); + assertInstanceOf(SAXException.class, cause); + } + } +} diff --git a/nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype-entity.xml b/nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype-entity.xml new file mode 100644 index 0000000000..ad770b1073 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype-entity.xml @@ -0,0 +1,16 @@ + %entity;]> + +&entity; diff --git a/nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype.xml b/nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype.xml new file mode 100644 index 0000000000..d6f400e7c0 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/resources/standard-document-doctype.xml @@ -0,0 +1,16 @@ + + + diff --git a/nifi-commons/nifi-xml-processing/src/test/resources/standard-document.xml b/nifi-commons/nifi-xml-processing/src/test/resources/standard-document.xml new file mode 100644 index 0000000000..72cca017d4 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/resources/standard-document.xml @@ -0,0 +1,16 @@ + + + diff --git a/nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document-doctype-entity.xml b/nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document-doctype-entity.xml new file mode 100644 index 0000000000..0b76a7578b --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document-doctype-entity.xml @@ -0,0 +1,16 @@ + %entity;]> + +&entity; diff --git a/nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document.xml b/nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document.xml new file mode 100644 index 0000000000..6dc65e802b --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/resources/standard-namespace-document.xml @@ -0,0 +1,16 @@ + + + diff --git a/nifi-commons/nifi-xml-processing/src/test/resources/standard-schema.xsd b/nifi-commons/nifi-xml-processing/src/test/resources/standard-schema.xsd new file mode 100644 index 0000000000..ba850e75f0 --- /dev/null +++ b/nifi-commons/nifi-xml-processing/src/test/resources/standard-schema.xsd @@ -0,0 +1,18 @@ + + + + + diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml index c2f8c69321..828ec93868 100644 --- a/nifi-commons/pom.xml +++ b/nifi-commons/pom.xml @@ -65,5 +65,6 @@ nifi-vault-utils nifi-web-utils nifi-write-ahead-log + nifi-xml-processing diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/authorization/AbstractPolicyBasedAuthorizer.java b/nifi-framework-api/src/main/java/org/apache/nifi/authorization/AbstractPolicyBasedAuthorizer.java index 6a4fa17057..76b8bc4236 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/authorization/AbstractPolicyBasedAuthorizer.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/authorization/AbstractPolicyBasedAuthorizer.java @@ -28,6 +28,7 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; +import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -36,6 +37,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -410,7 +412,7 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer final PoliciesUsersAndGroups policiesUsersAndGroups = parsePoliciesUsersAndGroups(fingerprint); if (isInheritable(policiesUsersAndGroups)) { - logger.debug("Inheriting Polciies, Users & Groups"); + logger.debug("Inheriting Policies, Users & Groups"); inheritPoliciesUsersAndGroups(policiesUsersAndGroups); } else { logger.info("Cannot directly inherit Policies, Users & Groups. Will backup existing Policies, Users & Groups, and then replace with proposed configuration"); @@ -427,8 +429,7 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = createSafeDocumentBuilder(); - final Document document = docBuilder.parse(in); + final Document document = parseFingerprint(in); final Element rootElement = document.getDocumentElement(); // parse all the users and add them to the current authorizer @@ -451,14 +452,14 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer Node policyNode = policyNodes.item(i); accessPolicies.add(parsePolicy((Element) policyNode)); } - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } return new PoliciesUsersAndGroups(accessPolicies, users, groups); } - public static DocumentBuilder createSafeDocumentBuilder() throws ParserConfigurationException { + private Document parseFingerprint(final InputStream inputStream) throws IOException { final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setSchema(null); docFactory.setNamespaceAware(true); @@ -471,7 +472,13 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer docFactory.setXIncludeAware(false); docFactory.setExpandEntityReferences(false); - return docFactory.newDocumentBuilder(); + try { + docFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + final DocumentBuilder documentBuilder = docFactory.newDocumentBuilder(); + return documentBuilder.parse(inputStream); + } catch (final ParserConfigurationException|SAXException e) { + throw new IOException("Fingerprint parsing failed", e); + } } private User parseUser(final Element element) { diff --git a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/pom.xml b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/pom.xml index a93a9f7d24..68a0faf8b6 100644 --- a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/pom.xml +++ b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/pom.xml @@ -58,6 +58,12 @@ 1.17.0-SNAPSHOT test + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + test + org.mockito mockito-junit-jupiter diff --git a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java index 6e1fd99d35..bae9316189 100644 --- a/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java +++ b/nifi-nar-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java @@ -28,10 +28,10 @@ import org.apache.nifi.processors.evtx.parser.FileHeaderFactory; import org.apache.nifi.processors.evtx.parser.MalformedChunkException; import org.apache.nifi.processors.evtx.parser.Record; import org.apache.nifi.processors.evtx.parser.bxml.RootNode; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -43,11 +43,8 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLStreamException; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -59,8 +56,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; @@ -159,7 +156,7 @@ public class ParseEvtxTest { } @Test - public void testProcessFileGranularity() throws IOException, MalformedChunkException, XMLStreamException { + public void testProcessFileGranularity() throws IOException, MalformedChunkException { String basename = "basename"; int chunkNum = 5; int offset = 10001; @@ -203,7 +200,7 @@ public class ParseEvtxTest { } @Test - public void testProcessChunkGranularity() throws IOException, MalformedChunkException, XMLStreamException { + public void testProcessChunkGranularity() throws IOException, MalformedChunkException { String basename = "basename"; int chunkNum = 5; int offset = 10001; @@ -266,7 +263,7 @@ public class ParseEvtxTest { } @Test - public void testProcess1RecordGranularity() throws IOException, MalformedChunkException, XMLStreamException { + public void testProcess1RecordGranularity() throws IOException, MalformedChunkException { String basename = "basename"; int chunkNum = 5; int offset = 10001; @@ -340,7 +337,7 @@ public class ParseEvtxTest { } @Test - public void fileGranularityLifecycleTest() throws IOException, ParserConfigurationException, SAXException { + public void fileGranularityLifecycleTest() throws IOException { String baseName = "testFileName"; String name = baseName + ".evtx"; TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class); @@ -374,7 +371,7 @@ public class ParseEvtxTest { } @Test - public void chunkGranularityLifecycleTest() throws IOException, ParserConfigurationException, SAXException { + public void chunkGranularityLifecycleTest() throws IOException { String baseName = "testFileName"; String name = baseName + ".evtx"; TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class); @@ -406,7 +403,7 @@ public class ParseEvtxTest { } @Test - public void recordGranularityLifecycleTest() throws IOException, ParserConfigurationException, SAXException { + public void recordGranularityLifecycleTest() throws IOException { String baseName = "testFileName"; String name = baseName + ".evtx"; TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class); @@ -471,12 +468,13 @@ public class ParseEvtxTest { testRunner.assertTransferCount(ParseEvtx.REL_SUCCESS, expectedCount); } - private int validateFlowFiles(List successFlowFiles) throws SAXException, IOException, ParserConfigurationException { + private int validateFlowFiles(List successFlowFiles) { assertTrue(successFlowFiles.size() > 0); int totalSize = 0; for (MockFlowFile successFlowFile : successFlowFiles) { // Verify valid XML output - Document document = XmlUtils.createSafeDocumentBuilder(false).parse(new ByteArrayInputStream(successFlowFile.toByteArray())); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + Document document = documentProvider.parse(successFlowFile.getContentStream()); Element documentElement = document.getDocumentElement(); assertEquals(XmlRootNodeHandler.EVENTS, documentElement.getTagName()); NodeList eventNodes = documentElement.getChildNodes(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java index 88bb5ddde3..804c7f2f40 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java @@ -26,9 +26,11 @@ import org.apache.nifi.authorization.generated.Property; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.properties.SensitivePropertyProviderFactoryAware; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.classloader.ClassLoaderUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.DisposableBean; @@ -40,7 +42,6 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; @@ -211,12 +212,13 @@ public class AuthorizerFactoryBean extends SensitivePropertyProviderFactoryAware final Schema schema = schemaFactory.newSchema(Authorizers.class.getResource(AUTHORIZERS_XSD)); // attempt to unmarshal - final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(authorizersConfigurationFile)); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(authorizersConfigurationFile)); final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(schema); final JAXBElement element = unmarshaller.unmarshal(xsr, Authorizers.class); return element.getValue(); - } catch (XMLStreamException | SAXException | JAXBException e) { + } catch (final ProcessingException | SAXException | JAXBException e) { throw new Exception("Unable to load the authorizer configuration file at: " + authorizersConfigurationFile.getAbsolutePath(), e); } } else { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/pom.xml index c8331fc8d2..27bc986ae4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/pom.xml @@ -47,6 +47,11 @@ nifi-utils test + + org.apache.nifi + nifi-xml-processing + test + commons-io commons-io diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/XmlValidator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/XmlValidator.java index 4cc6b22589..d75c3602a4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/XmlValidator.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/test/java/org/apache/nifi/documentation/html/XmlValidator.java @@ -16,13 +16,13 @@ */ package org.apache.nifi.documentation.html; -import java.io.IOException; -import java.io.StringReader; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.junit.Assert; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; /** * A helper class to validate xml documents. @@ -30,21 +30,21 @@ import org.xml.sax.SAXException; * */ public class XmlValidator { + private static final String DOCTYPE = ""; + + private static final String EMPTY = ""; /** - * Asserts a failure if the provided XML is not valid. This method does - * not use the "safe" {@link DocumentBuilderFactory} from - * {@code XmlUtils#createSafeDocumentBuilder(Schema, boolean)} because it checks - * generated documentation which contains a doctype. + * Asserts a failure if the provided XHTML is not valid * * @param xml the XML to validate */ public static void assertXmlValid(String xml) { + final String html = xml.replace(DOCTYPE, EMPTY); try { - final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - dbf.setNamespaceAware(true); - dbf.newDocumentBuilder().parse(new InputSource(new StringReader(xml))); - } catch (SAXException | IOException | ParserConfigurationException e) { + final DocumentProvider provider = new StandardDocumentProvider(); + provider.parse(new ByteArrayInputStream(html.getBytes(StandardCharsets.UTF_8))); + } catch (final ProcessingException e) { Assert.fail(e.getMessage()); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/pom.xml index 0ed15c34ca..6d9a587f69 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/pom.xml @@ -204,6 +204,11 @@ org.apache.nifi nifi-security-utils + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.nifi nifi-properties diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java index dcd23d7dc3..5be8335a79 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java @@ -30,13 +30,16 @@ import org.apache.nifi.authorization.resource.ResourceType; import org.apache.nifi.authorization.util.IdentityMapping; import org.apache.nifi.authorization.util.IdentityMappingUtil; import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.user.generated.Users; import org.apache.nifi.util.FlowInfo; import org.apache.nifi.util.FlowParser; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.FileUtils; import org.apache.nifi.web.api.dto.PortDTO; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -51,8 +54,6 @@ import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; @@ -501,8 +502,8 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = docBuilder.parse(in); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); // parse all the policies and add them to the current access policy provider @@ -511,7 +512,7 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide Node policyNode = policyNodes.item(i); policies.add(parsePolicy((Element) policyNode)); } - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } @@ -631,13 +632,14 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide private Authorizations unmarshallAuthorizations() throws JAXBException { try { - final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(authorizationsFile)); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(authorizationsFile)); final Unmarshaller unmarshaller = JAXB_AUTHORIZATIONS_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(authorizationsSchema); final JAXBElement element = unmarshaller.unmarshal(xsr, Authorizations.class); return element.getValue(); - } catch (XMLStreamException e) { + } catch (final ProcessingException e) { logger.error("Encountered an error reading authorizations file: ", e); throw new JAXBException("Error reading authorizations file", e); } @@ -757,8 +759,9 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide final XMLStreamReader xsr; try { - xsr = XmlUtils.createSafeReader(new StreamSource(authorizedUsersFile)); - } catch (XMLStreamException e) { + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile)); + } catch (final ProcessingException e) { logger.error("Encountered an error reading authorized users file: ", e); throw new JAXBException("Error reading authorized users file", e); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java index f59f718dbe..855e80a331 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java @@ -28,16 +28,18 @@ import org.apache.nifi.authorization.file.tenants.generated.Users; import org.apache.nifi.authorization.util.IdentityMapping; import org.apache.nifi.authorization.util.IdentityMappingUtil; import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.FileUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; import javax.xml.XMLConstants; import javax.xml.bind.JAXBContext; @@ -45,8 +47,6 @@ import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; @@ -598,8 +598,8 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = docBuilder.parse(in); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); // parse all the users and add them to the current user group provider @@ -615,7 +615,7 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { Node groupNode = groupNodes.item(i); groups.add(parseGroup((Element) groupNode)); } - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } @@ -724,11 +724,12 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { final Unmarshaller unmarshaller = JAXB_TENANTS_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(tenantsSchema); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); try { - final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(tenantsFile)); + final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(tenantsFile)); final JAXBElement element = unmarshaller.unmarshal(xsr, Tenants.class); return element.getValue(); - } catch (XMLStreamException e) { + } catch (final ProcessingException e) { throw new JAXBException("Error unmarshalling tenants", e); } } @@ -752,10 +753,11 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { throw new AuthorizerCreationException("Legacy Authorized Users File '" + legacyAuthorizedUsersFile + "' does not exists"); } - XMLStreamReader xsr; + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader xsr; try { - xsr = XmlUtils.createSafeReader(new StreamSource(authorizedUsersFile)); - } catch (XMLStreamException e) { + xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile)); + } catch (final ProcessingException e) { throw new AuthorizerCreationException("Error converting the legacy authorizers file", e); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/groovy/org/apache/nifi/authorization/FlowParserTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/groovy/org/apache/nifi/authorization/FlowParserTest.groovy deleted file mode 100644 index fbf4b16b58..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/groovy/org/apache/nifi/authorization/FlowParserTest.groovy +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.authorization - -import org.apache.nifi.util.FlowParser -import org.junit.After -import org.junit.Before -import org.junit.BeforeClass -import org.junit.Test -import org.junit.runner.RunWith -import org.junit.runners.JUnit4 -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -@RunWith(JUnit4.class) -class FlowParserTest extends GroovyTestCase { - private static final Logger logger = LoggerFactory.getLogger(FlowParserTest.class) - - @BeforeClass - static void setUpOnce() throws Exception { - logger.metaClass.methodMissing = { String name, args -> - logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") - } - } - - @Before - void setUp() throws Exception { - - } - - @After - void tearDown() throws Exception { - - } - - @Test - void testShouldHandleXXEInDocumentBuilder() { - // Arrange - final String XXE_TEMPLATE_FILEPATH = "src/test/resources/flow-with-xxe.xml.gz" - - FlowParser fp = new FlowParser() - - // Act - def parsedFlow = fp.parse(new File(XXE_TEMPLATE_FILEPATH)) - logger.info("Parsed ${parsedFlow.toString()}") - - // Assert - - // The existing logic logs & swallows any exceptions and returns null - assert !parsedFlow - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-xxe.xml.gz b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-xxe.xml.gz deleted file mode 100644 index 241f0047536cdf35acbb976e32602635cb245d43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 794 zcmV+#1Lgc5iwFo8XCGex17>V*cP)2mbZ9Mjcx5hlZEOJDRoiZxKoEV;S8RC4fX&6O z2vHO}k)_75Y|<*Ks%n98vMSz1yG!Eq>pNf|zQmQ<$G(8(%*@W2+irNdS?A;hq+)_! z7*5YN2=G*7j6Yo%5AlsTG>FoaXEYZaE(|V=%P2e_-(Sa%^9kXgRl&%&@O(0hr|~1% zY=A5um+|C|7%P^;C(E)xr&d9gdd&{^suM~)p@r?rjm1Yv!w^=j7>%F0y=p>L%qi%LS7Wf6D-8yq(^X_l@` z*Gcji@(D=rHiTG_=rg>%r+7q8Ynkl$a2F^J;ELu7OR%Pw7#$>{u#BrJNO~vMXBdP>fJ@?uY%0+ zZQo4nk!`vO1SaldA3sT8uM*Gm2F~#!#Mg5&t&aP{3OGZrH*kI5@kf39^&P+72j3s| zhE8DH1G^u%uIm{`yRfqz2Yp#Q%x2Ts?a6-q^}v@33rm}OHIec1;90He~4Pyhe` diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/pom.xml index 17304fe619..583756de0e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/pom.xml @@ -42,6 +42,11 @@ nifi-security-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.springframework.security spring-security-core diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/src/main/java/org/apache/nifi/authorization/StandardManagedAuthorizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/src/main/java/org/apache/nifi/authorization/StandardManagedAuthorizer.java index b6d09298e4..d115794fd7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/src/main/java/org/apache/nifi/authorization/StandardManagedAuthorizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization-providers/src/main/java/org/apache/nifi/authorization/StandardManagedAuthorizer.java @@ -22,15 +22,13 @@ import org.apache.nifi.authorization.exception.AuthorizerCreationException; import org.apache.nifi.authorization.exception.AuthorizerDestructionException; import org.apache.nifi.authorization.exception.UninheritableAuthorizationsException; import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; @@ -239,12 +237,13 @@ public class StandardManagedAuthorizer implements ManagedAuthorizer { } } - private final FingerprintHolder parseFingerprint(final String fingerprint) throws AuthorizationAccessException { + private FingerprintHolder parseFingerprint(final String fingerprint) throws AuthorizationAccessException { final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true); - final Document document = docBuilder.parse(in); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setNamespaceAware(true); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); final NodeList accessPolicyProviderList = rootElement.getElementsByTagName(ACCESS_POLICY_PROVIDER_ELEMENT); @@ -260,7 +259,7 @@ public class StandardManagedAuthorizer implements ManagedAuthorizer { final Node accessPolicyProvider = accessPolicyProviderList.item(0); final Node userGroupProvider = userGroupProviderList.item(0); return new FingerprintHolder(accessPolicyProvider.getTextContent(), userGroupProvider.getTextContent()); - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/pom.xml index 6a8a44aa50..ce4f091cab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/pom.xml @@ -47,6 +47,16 @@ org.apache.nifi nifi-framework-core-api + + org.apache.nifi + nifi-security-utils + 1.17.0-SNAPSHOT + + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java index 16aac26ce4..7e6865ac1d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java @@ -17,15 +17,17 @@ package org.apache.nifi.cluster.protocol; import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; @@ -122,9 +124,10 @@ public class HeartbeatPayload { public static HeartbeatPayload unmarshal(final InputStream is) throws ProtocolException { try { final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - final XMLStreamReader xsr = XmlUtils.createSafeReader(is); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(is)); return (HeartbeatPayload) unmarshaller.unmarshal(xsr); - } catch (final JAXBException | XMLStreamException e) { + } catch (final JAXBException | ProcessingException e) { throw new ProtocolException(e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java index 52d8dff54e..30cce26b5c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java @@ -22,21 +22,17 @@ import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; import org.apache.nifi.cluster.protocol.jaxb.message.DataFlowAdapter; import org.apache.nifi.controller.flow.VersionedDataflow; import org.apache.nifi.controller.serialization.FlowSerializationException; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; -import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; import org.xml.sax.helpers.DefaultHandler; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.Serializable; -import java.net.URL; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -48,7 +44,6 @@ import java.util.Set; */ @XmlJavaTypeAdapter(DataFlowAdapter.class) public class StandardDataFlow implements Serializable, DataFlow { - private static final URL FLOW_XSD_RESOURCE = StandardDataFlow.class.getClassLoader().getResource("/FlowConfiguration.xsd"); private static final Logger logger = LoggerFactory.getLogger(StandardDataFlow.class); private static final long serialVersionUID = 1L; @@ -135,21 +130,19 @@ public class StandardDataFlow implements Serializable, DataFlow { return null; } - // create document by parsing proposed flow bytes try { - // create validating document builder - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true); - docBuilder.setErrorHandler(new DefaultHandler() { + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setNamespaceAware(true); + documentProvider.setErrorHandler(new DefaultHandler() { @Override public void error(final SAXParseException e) { logger.warn("Schema validation error parsing Flow Configuration at line {}, col {}: {}", e.getLineNumber(), e.getColumnNumber(), e.getMessage()); } }); - // parse flow - return docBuilder.parse(new ByteArrayInputStream(flow)); - } catch (final SAXException | ParserConfigurationException | IOException ex) { - throw new FlowSerializationException(ex); + return documentProvider.parse(new ByteArrayInputStream(flow)); + } catch (final ProcessingException e) { + throw new FlowSerializationException("Flow parsing failed", e); } } @@ -163,8 +156,7 @@ public class StandardDataFlow implements Serializable, DataFlow { objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - final VersionedDataflow versionedDataflow = objectMapper.readValue(flow, VersionedDataflow.class); - return versionedDataflow; + return objectMapper.readValue(flow, VersionedDataflow.class); } catch (final Exception e) { throw new FlowSerializationException("Could not parse flow as a VersionedDataflow", e); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java index 34da349821..2baade5f6c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java @@ -19,14 +19,16 @@ package org.apache.nifi.cluster.protocol.jaxb; import org.apache.nifi.cluster.protocol.ProtocolContext; import org.apache.nifi.cluster.protocol.ProtocolMessageMarshaller; import org.apache.nifi.cluster.protocol.ProtocolMessageUnmarshaller; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -137,10 +139,11 @@ public class JaxbProtocolContext implements ProtocolContext { final Unmarshaller unmarshaller = jaxbCtx.createUnmarshaller(); final byte[] msg = new byte[totalBytesRead]; buffer.get(msg); - final XMLStreamReader xsr = XmlUtils.createSafeReader(new ByteArrayInputStream(msg)); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(new ByteArrayInputStream(msg))); return (T) unmarshaller.unmarshal(xsr); - } catch (final JAXBException | XMLStreamException e) { + } catch (final JAXBException | ProcessingException e) { throw new IOException("Failed unmarshalling protocol message due to: " + e, e); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml index b9d0ccc707..21c17aa189 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml @@ -81,6 +81,16 @@ nifi-external-resource-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-security-utils + 1.17.0-SNAPSHOT + + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.nifi.registry diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/state/config/StateManagerConfiguration.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/state/config/StateManagerConfiguration.java index 9795a72955..fd377c9459 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/state/config/StateManagerConfiguration.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/state/config/StateManagerConfiguration.java @@ -18,21 +18,22 @@ package org.apache.nifi.controller.state.config; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import org.apache.nifi.components.state.Scope; import org.apache.nifi.controller.state.ConfigParseException; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.DomUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.SAXException; public class StateManagerConfiguration { private final Map providers; @@ -62,11 +63,10 @@ public class StateManagerConfiguration { public static StateManagerConfiguration parse(final File configFile) throws IOException, ConfigParseException { final Document document; - DocumentBuilder builder; - try { - builder = XmlUtils.createSafeDocumentBuilder(false); - document = builder.parse(configFile); - } catch (ParserConfigurationException | SAXException e) { + try (final InputStream inputStream = new FileInputStream(configFile)) { + final DocumentProvider documentProvider = new StandardDocumentProvider(); + document = documentProvider.parse(inputStream); + } catch (final ProcessingException e) { throw new ConfigParseException("Unable to parse file " + configFile + ", as it does not appear to be a valid XML File", e); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml index e24bdb228d..d06bfeb8f6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml @@ -78,6 +78,11 @@ nifi-repository-encryption 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.nifi nifi-properties diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java index e8a33a5b55..8c9bc0a3df 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java @@ -27,12 +27,15 @@ import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; + import org.apache.nifi.cluster.protocol.ProtocolException; import org.apache.nifi.jaxb.BulletinAdapter; import org.apache.nifi.reporting.Bulletin; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; /** * The payload of the bulletins. @@ -80,9 +83,10 @@ public class BulletinsPayload { public static BulletinsPayload unmarshal(final InputStream is) throws ProtocolException { try { final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - final XMLStreamReader xsr = XmlUtils.createSafeReader(is); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader xsr = provider.getStreamReader(new StreamSource(is)); return (BulletinsPayload) unmarshaller.unmarshal(xsr); - } catch (final JAXBException | XMLStreamException e) { + } catch (final JAXBException | ProcessingException e) { throw new ProtocolException(e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java index 202c48f176..549b3b4f4b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java @@ -48,16 +48,15 @@ import org.apache.nifi.registry.flow.FlowRegistryClient; import org.apache.nifi.registry.flow.VersionControlInformation; import org.apache.nifi.remote.PublicPort; import org.apache.nifi.remote.RemoteGroupPort; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.CharacterFilterUtils; import org.apache.nifi.util.StringUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; @@ -93,8 +92,9 @@ public class StandardFlowSerializer implements FlowSerializer { public Document transform(final FlowController controller, final ScheduledStateLookup scheduledStateLookup) throws FlowSerializationException { try { // create a new, empty document - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true); - final Document doc = docBuilder.newDocument(); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setNamespaceAware(true); + final Document doc = documentProvider.newDocument(); // populate document with controller state final Element rootNode = doc.createElement("flowController"); @@ -127,7 +127,7 @@ public class StandardFlowSerializer implements FlowSerializer { } return doc; - } catch (final ParserConfigurationException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) { + } catch (final ProcessingException | DOMException | TransformerFactoryConfigurationError | IllegalArgumentException e) { throw new FlowSerializationException(e); } } @@ -685,10 +685,11 @@ public class StandardFlowSerializer implements FlowSerializer { try { final byte[] serialized = TemplateSerializer.serialize(template.getDetails()); - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(true); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setNamespaceAware(true); final Document document; try (final InputStream in = new ByteArrayInputStream(serialized)) { - document = docBuilder.parse(in); + document = documentProvider.parse(in); } final Node templateNode = element.getOwnerDocument().importNode(document.getDocumentElement(), true); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java index b1475fd00a..30261cf13b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java @@ -24,11 +24,12 @@ import org.apache.nifi.controller.serialization.FlowFromDOMFactory; import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.reporting.BulletinRepository; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.BundleUtils; import org.apache.nifi.util.DomUtils; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -36,8 +37,6 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -60,9 +59,9 @@ public class ControllerServiceLoader { final PropertyEncryptor encryptor, final BulletinRepository bulletinRepo, final boolean autoResumeState, final FlowEncodingVersion encodingVersion) throws IOException { try (final InputStream in = new BufferedInputStream(serializedStream)) { - final DocumentBuilder builder = XmlUtils.createSafeDocumentBuilder(null); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); - builder.setErrorHandler(new org.xml.sax.ErrorHandler() { + documentProvider.setErrorHandler(new org.xml.sax.ErrorHandler() { @Override public void fatalError(final SAXParseException err) throws SAXException { @@ -92,15 +91,15 @@ public class ControllerServiceLoader { } }); - final Document document = builder.parse(in); + final Document document = documentProvider.parse(in); final Element controllerServices = document.getDocumentElement(); final List serviceElements = DomUtils.getChildElementsByTagName(controllerServices, "controllerService"); final Map controllerServiceMap = ControllerServiceLoader.loadControllerServices(serviceElements, controller, parentGroup, encryptor, encodingVersion); enableControllerServices(controllerServiceMap, controller, encryptor, autoResumeState, encodingVersion); return new ArrayList<>(controllerServiceMap.keySet()); - } catch (SAXException | ParserConfigurationException sxe) { - throw new IOException(sxe); + } catch (final ProcessingException e) { + throw new IOException("Parsing Controller Services failed", e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java index abee7c26ac..4aeb52661d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java @@ -26,27 +26,26 @@ import org.apache.nifi.controller.serialization.FlowFromDOMFactory; import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.encrypt.SensitiveValueEncoder; import org.apache.nifi.nar.ExtensionManager; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.BundleUtils; import org.apache.nifi.util.DomUtils; import org.apache.nifi.util.LoggingXmlParserErrorHandler; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.ReportingTaskDTO; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; +import org.xml.sax.ErrorHandler; import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -83,7 +82,7 @@ public class FingerprintFactory { private static final String ENCRYPTED_VALUE_PREFIX = "enc{"; private static final String ENCRYPTED_VALUE_SUFFIX = "}"; private final PropertyEncryptor encryptor; - private final DocumentBuilder flowConfigDocBuilder; + private final Schema schema; private final ExtensionManager extensionManager; private final SensitiveValueEncoder sensitiveValueEncoder; @@ -95,25 +94,11 @@ public class FingerprintFactory { this.sensitiveValueEncoder = sensitiveValueEncoder; final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - final Schema schema; try { schema = schemaFactory.newSchema(FingerprintFactory.class.getResource(FLOW_CONFIG_XSD)); } catch (final Exception e) { throw new RuntimeException("Failed to parse schema for file flow configuration.", e); } - try { - flowConfigDocBuilder = XmlUtils.createSafeDocumentBuilder(schema, true); - flowConfigDocBuilder.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger)); - } catch (final Exception e) { - throw new RuntimeException("Failed to create document builder for flow configuration.", e); - } - } - - public FingerprintFactory(final PropertyEncryptor encryptor, final DocumentBuilder docBuilder, final ExtensionManager extensionManager, final SensitiveValueEncoder sensitiveValueEncoder) { - this.encryptor = encryptor; - this.flowConfigDocBuilder = docBuilder; - this.extensionManager = extensionManager; - this.sensitiveValueEncoder = sensitiveValueEncoder; } /** @@ -183,9 +168,15 @@ public class FingerprintFactory { } try { - return flowConfigDocBuilder.parse(new ByteArrayInputStream(flow)); - } catch (final SAXException | IOException ex) { - throw new FingerprintException(ex); + final ErrorHandler errorHandler = new LoggingXmlParserErrorHandler("Flow Configuration", logger); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setSchema(schema); + documentProvider.setNamespaceAware(true); + documentProvider.setErrorHandler(errorHandler); + + return documentProvider.parse(new ByteArrayInputStream(flow)); + } catch (final ProcessingException e) { + throw new FingerprintException("Flow Parsing failed", e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardSnippetDeserializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardSnippetDeserializer.java index 9aba5104f1..50b204232d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardSnippetDeserializer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardSnippetDeserializer.java @@ -21,11 +21,14 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; + import org.apache.nifi.controller.StandardSnippet; import org.apache.nifi.controller.serialization.FlowSerializationException; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; public class StandardSnippetDeserializer { @@ -33,10 +36,11 @@ public class StandardSnippetDeserializer { try { JAXBContext context = JAXBContext.newInstance(StandardSnippet.class); Unmarshaller unmarshaller = context.createUnmarshaller(); - XMLStreamReader xsr = XmlUtils.createSafeReader(inStream); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + XMLStreamReader xsr = provider.getStreamReader(new StreamSource(inStream)); JAXBElement snippetElement = unmarshaller.unmarshal(xsr, StandardSnippet.class); return snippetElement.getValue(); - } catch (final JAXBException | XMLStreamException e) { + } catch (final JAXBException | ProcessingException e) { throw new FlowSerializationException(e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java index 43e89ccf91..c62a987fa8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java @@ -17,14 +17,15 @@ package org.apache.nifi.persistence; import org.apache.nifi.controller.serialization.FlowSerializationException; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.web.api.dto.TemplateDTO; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.transform.stream.StreamSource; import java.io.InputStream; @@ -46,14 +47,16 @@ public class TemplateDeserializer { } public static TemplateDTO deserialize(final StreamSource source) { + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + try { - final XMLStreamReader xsr = XmlUtils.createSafeReader(source); + final XMLStreamReader xsr = provider.getStreamReader(source); final Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); final JAXBElement templateElement = unmarshaller.unmarshal(xsr, TemplateDTO.class); final TemplateDTO templateDto = templateElement.getValue(); return templateDto; - } catch (final JAXBException | XMLStreamException e) { + } catch (final JAXBException | ProcessingException e) { throw new FlowSerializationException(e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java index d46e2ffacf..cc1d652eac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java @@ -22,20 +22,21 @@ import org.apache.nifi.controller.flow.VersionedDataflow; import org.apache.nifi.controller.serialization.FlowFromDOMFactory; import org.apache.nifi.flow.VersionedPort; import org.apache.nifi.flow.VersionedProcessGroup; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.file.FileUtils; import org.apache.nifi.web.api.dto.PortDTO; import org.apache.nifi.web.api.dto.PositionDTO; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.xml.sax.ErrorHandler; import org.xml.sax.SAXException; import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Result; import javax.xml.transform.Source; @@ -117,7 +118,7 @@ public class FlowParser { return parseJson(flowBytes); } catch (final SAXException | ParserConfigurationException | IOException ex) { - logger.error("Unable to parse flow {} due to {}", new Object[] { flowPath.toAbsolutePath(), ex }); + logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), ex); return null; } } @@ -133,11 +134,14 @@ public class FlowParser { private FlowInfo parseXml(final byte[] flowBytes) throws ParserConfigurationException, IOException, SAXException { // create validating document builder - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(flowSchema); - docBuilder.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger)); + final ErrorHandler errorHandler = new LoggingXmlParserErrorHandler("Flow Configuration", logger); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setSchema(flowSchema); + documentProvider.setNamespaceAware(true); + documentProvider.setErrorHandler(errorHandler); // parse the flow - final Document document = docBuilder.parse(new ByteArrayInputStream(flowBytes)); + final Document document = documentProvider.parse(new ByteArrayInputStream(flowBytes)); // extract the root group id final Element rootElement = document.getDocumentElement(); @@ -236,11 +240,13 @@ public class FlowParser { } // create validating document builder - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(flowSchema); - docBuilder.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger)); - return docBuilder.parse(new ByteArrayInputStream(flowBytes)); - } catch (final SAXException | ParserConfigurationException | IOException ex) { - logger.error("Unable to parse flow {} due to {}", new Object[]{flowPath.toAbsolutePath(), ex}); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger)); + documentProvider.setSchema(flowSchema); + documentProvider.setNamespaceAware(true); + return documentProvider.parse(new ByteArrayInputStream(flowBytes)); + } catch (final ProcessingException | IOException e) { + logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), e); return null; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java index f8961d86b9..96ee449ff7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java @@ -33,21 +33,14 @@ import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.StandardExtensionDiscoveringManager; import org.apache.nifi.remote.RemoteGroupPort; import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.junit.Before; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.ErrorHandler; -import org.xml.sax.SAXException; -import org.xml.sax.SAXParseException; -import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; -import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; @@ -55,7 +48,6 @@ import java.util.Collections; import java.util.Optional; import static org.apache.nifi.controller.serialization.ScheduledStateLookup.IDENTITY_LOOKUP; -import static org.apache.nifi.fingerprint.FingerprintFactory.FLOW_CONFIG_XSD; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; @@ -170,12 +162,11 @@ public class FingerprintFactoryTest { } @Test - public void testPublicPortWithDifferentFingerprintInAccessPolicies() throws IOException, ParserConfigurationException, SAXException { + public void testPublicPortWithDifferentFingerprintInAccessPolicies() throws IOException { final String f1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null); assertEquals(2, StringUtils.countMatches(f1, "user1group1")); - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = docBuilder.parse(new File("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml")); + final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml"); final Element rootProcessGroup = document.getDocumentElement(); final StringBuilder sb = new StringBuilder(); @@ -190,9 +181,8 @@ public class FingerprintFactoryTest { } @Test - public void testPublicPortWithNoAccessPoliciesFingerprint() throws ParserConfigurationException, IOException, SAXException { - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = docBuilder.parse(new File("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml")); + public void testPublicPortWithNoAccessPoliciesFingerprint() throws IOException { + final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml"); final Element rootProcessGroup = document.getDocumentElement(); final StringBuilder sb = new StringBuilder(); @@ -203,52 +193,21 @@ public class FingerprintFactoryTest { assertTrue(fingerprint.contains("NO_GROUP_ACCESS_CONTROL")); } - @Test - public void testSchemaValidation() throws IOException { - FingerprintFactory fp = new FingerprintFactory(null, getValidatingDocumentBuilder(), extensionManager, null); - fp.createFingerprint(getResourceBytes("/nifi/fingerprint/validating-flow.xml"), null); + private Document getDocument(final String filePath) throws IOException { + try (final FileInputStream inputStream = new FileInputStream(filePath)) { + final DocumentProvider documentProvider = new StandardDocumentProvider(); + return documentProvider.parse(inputStream); + } } private byte[] getResourceBytes(final String resource) throws IOException { return IOUtils.toByteArray(FingerprintFactoryTest.class.getResourceAsStream(resource)); } - private DocumentBuilder getValidatingDocumentBuilder() { - final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - final Schema schema; - try { - schema = schemaFactory.newSchema(FingerprintFactory.class.getResource(FLOW_CONFIG_XSD)); - } catch (final Exception e) { - throw new RuntimeException("Failed to parse schema for file flow configuration.", e); - } - try { - DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(schema, true); - docBuilder.setErrorHandler(new ErrorHandler() { - @Override - public void warning(SAXParseException e) throws SAXException { - throw e; - } - - @Override - public void error(SAXParseException e) throws SAXException { - throw e; - } - - @Override - public void fatalError(SAXParseException e) throws SAXException { - throw e; - } - }); - return docBuilder; - } catch (final Exception e) { - throw new RuntimeException("Failed to create document builder for flow configuration.", e); - } - } - private Element serializeElement(final PropertyEncryptor encryptor, final Class componentClass, final T component, final String serializerMethodName, ScheduledStateLookup scheduledStateLookup) throws Exception { - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document doc = docBuilder.newDocument(); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document doc = documentProvider.newDocument(); final FlowSerializer flowSerializer = new StandardFlowSerializer(encryptor); final Method serializeMethod = StandardFlowSerializer.class.getDeclaredMethod(serializerMethodName, @@ -394,9 +353,8 @@ public class FingerprintFactoryTest { } @Test - public void testControllerServicesIncludedInGroupFingerprint() throws ParserConfigurationException, IOException, SAXException { - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = docBuilder.parse(new File("src/test/resources/nifi/fingerprint/group-with-controller-services.xml")); + public void testControllerServicesIncludedInGroupFingerprint() throws IOException { + final Document document = getDocument("src/test/resources/nifi/fingerprint/group-with-controller-services.xml"); final Element processGroup = document.getDocumentElement(); final StringBuilder sb = new StringBuilder(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java index c04b4ddf95..a3dbec19a1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java @@ -16,11 +16,12 @@ */ package org.apache.nifi.persistence; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.ComponentIdGenerator; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.TemplateDTO; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.eclipse.jgit.diff.DiffFormatter; import org.eclipse.jgit.diff.EditList; import org.eclipse.jgit.diff.HistogramDiff; @@ -32,6 +33,7 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.Unmarshaller; import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -69,7 +71,8 @@ public class TemplateSerializerTest { ByteArrayInputStream in = new ByteArrayInputStream(serTemplate); JAXBContext context = JAXBContext.newInstance(TemplateDTO.class); Unmarshaller unmarshaller = context.createUnmarshaller(); - XMLStreamReader xsr = XmlUtils.createSafeReader(in); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + XMLStreamReader xsr = provider.getStreamReader(new StreamSource(in)); JAXBElement templateElement = unmarshaller.unmarshal(xsr, TemplateDTO.class); TemplateDTO deserTemplate = templateElement.getValue(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/pom.xml index 10adde9488..5e8b4fe983 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/pom.xml @@ -195,6 +195,10 @@ org.apache.nifi nifi-nar-utils + + org.apache.nifi + nifi-xml-processing + org.apache.nifi nifi-web-security diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java index 50b132f0cd..f9955eaa96 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java @@ -34,6 +34,8 @@ import org.apache.nifi.processor.Relationship; import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.dao.ProcessorDAO; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; @@ -43,11 +45,9 @@ import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import java.io.StringReader; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.text.Collator; import java.util.ArrayList; import java.util.Collection; @@ -180,16 +180,12 @@ public class ProcessorAuditor extends NiFiAuditor { try { - InputSource is = new InputSource(); - is.setCharacterStream(new StringReader(newValue)); - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.parse(is); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + Document doc = documentProvider.parse(new ByteArrayInputStream(newValue.getBytes(StandardCharsets.UTF_8))); NodeList nList = doc.getChildNodes(); final Map xmlDumpNew = new HashMap<>(); getItemPaths(nList, ""+doc.getNodeName(), xmlDumpNew); - is.setCharacterStream(new StringReader(oldValue)); - doc = dBuilder.parse(is); + doc = documentProvider.parse(new ByteArrayInputStream(oldValue.getBytes(StandardCharsets.UTF_8))); nList = doc.getChildNodes(); final Map xmlDumpOld = new HashMap<>(); getItemPaths(nList, ""+doc.getNodeName(), xmlDumpOld); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java index c04204b9d8..1c946c72ba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java @@ -78,6 +78,8 @@ import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.nifi.authorization.AuthorizableLookup; @@ -113,7 +115,6 @@ import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.registry.variable.VariableRegistryUpdateRequest; import org.apache.nifi.registry.variable.VariableRegistryUpdateStep; import org.apache.nifi.remote.util.SiteToSiteRestApiClient; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.Revision; import org.apache.nifi.web.api.dto.AffectedComponentDTO; @@ -173,6 +174,8 @@ import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.LongParameter; import org.apache.nifi.web.security.token.NiFiAuthenticationToken; import org.apache.nifi.web.util.Pause; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -3814,7 +3817,8 @@ public class ProcessGroupResource extends FlowUpdateResource templateElement = unmarshaller.unmarshal(xsr, TemplateDTO.class); template = templateElement.getValue(); } catch (JAXBException jaxbe) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java index 75120aeede..f8c9967d9e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/spring/LoginIdentityProviderFactoryBean.java @@ -33,8 +33,9 @@ import org.apache.nifi.bundle.Bundle; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.properties.SensitivePropertyProviderFactoryAware; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.FactoryBean; import org.xml.sax.SAXException; @@ -138,7 +139,8 @@ public class LoginIdentityProviderFactoryBean extends SensitivePropertyProviderF final Schema schema = schemaFactory.newSchema(LoginIdentityProviders.class.getResource(LOGIN_IDENTITY_PROVIDERS_XSD)); // attempt to unmarshal - XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(loginIdentityProvidersConfigurationFile)); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + XMLStreamReader xsr = provider.getStreamReader(new StreamSource(loginIdentityProvidersConfigurationFile)); final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(schema); final JAXBElement element = unmarshaller.unmarshal(xsr, LoginIdentityProviders.class); diff --git a/nifi-nar-bundles/nifi-framework-bundle/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/pom.xml index 62d95da343..53dbfc843a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/pom.xml @@ -60,6 +60,11 @@ nifi-security-utils-api 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.nifi nifi-expression-language diff --git a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml index 37a73f8243..d7525e7bf7 100644 --- a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml +++ b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml @@ -186,7 +186,7 @@ org.apache.nifi - nifi-security-utils + nifi-xml-processing 1.17.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/ManagedRangerAuthorizer.java b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/ManagedRangerAuthorizer.java index f427346d55..4f5f340ff6 100644 --- a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/ManagedRangerAuthorizer.java +++ b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/src/main/java/org/apache/nifi/ranger/authorization/ManagedRangerAuthorizer.java @@ -23,8 +23,6 @@ import java.io.IOException; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.Set; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; @@ -45,12 +43,12 @@ import org.apache.nifi.authorization.exception.AuthorizationAccessException; import org.apache.nifi.authorization.exception.AuthorizerCreationException; import org.apache.nifi.authorization.exception.AuthorizerDestructionException; import org.apache.nifi.authorization.exception.UninheritableAuthorizationsException; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements ManagedAuthorizer { private static final String USER_GROUP_PROVIDER_ELEMENT = "userGroupProvider"; @@ -128,8 +126,8 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man final StringWriter out = new StringWriter(); try { // create the document - final DocumentBuilder documentBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = documentBuilder.newDocument(); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.newDocument(); // create the root element final Element managedRangerAuthorizationsElement = document.createElement("managedRangerAuthorizations"); @@ -146,7 +144,7 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man final Transformer transformer = TransformerFactory.newInstance().newTransformer(); transformer.transform(new DOMSource(document), new StreamResult(out)); - } catch (ParserConfigurationException | TransformerException e) { + } catch (final ProcessingException | TransformerException e) { throw new AuthorizationAccessException("Unable to generate fingerprint", e); } @@ -192,8 +190,8 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = docBuilder.parse(in); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); final NodeList userGroupProviderList = rootElement.getElementsByTagName(USER_GROUP_PROVIDER_ELEMENT); @@ -203,7 +201,7 @@ public class ManagedRangerAuthorizer extends RangerNiFiAuthorizer implements Man final Node userGroupProvider = userGroupProviderList.item(0); return userGroupProvider.getTextContent(); - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } } diff --git a/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/pom.xml b/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/pom.xml index 8aba19891f..69da92cf39 100644 --- a/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/pom.xml +++ b/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/pom.xml @@ -46,5 +46,10 @@ nifi-single-user-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/src/main/java/org/apache/nifi/authorization/single/user/SingleUserAuthorizer.java b/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/src/main/java/org/apache/nifi/authorization/single/user/SingleUserAuthorizer.java index 9aeb130487..e78e386555 100644 --- a/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/src/main/java/org/apache/nifi/authorization/single/user/SingleUserAuthorizer.java +++ b/nifi-nar-bundles/nifi-single-user-iaa-providers-bundle/nifi-single-user-iaa-providers/src/main/java/org/apache/nifi/authorization/single/user/SingleUserAuthorizer.java @@ -26,14 +26,16 @@ import org.apache.nifi.authorization.annotation.AuthorizerContext; import org.apache.nifi.authorization.exception.AuthorizationAccessException; import org.apache.nifi.authorization.exception.AuthorizerCreationException; import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.StartElement; import javax.xml.stream.events.XMLEvent; +import javax.xml.transform.stream.StreamSource; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -149,9 +151,7 @@ public class SingleUserAuthorizer implements Authorizer { } private XMLEventReader getProvidersReader(final InputStream inputStream) throws XMLStreamException { - final XMLInputFactory inputFactory = XMLInputFactory.newFactory(); - inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - return inputFactory.createXMLEventReader(inputStream); + final XMLEventReaderProvider readerProvider = new StandardXMLEventReaderProvider(); + return readerProvider.getEventReader(new StreamSource(inputStream)); } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 6fcca80a52..01aeb4bcb2 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -158,6 +158,11 @@ nifi-security-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + com.hierynomus sshj diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java index ae5020689f..de1c05424b 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXPath.java @@ -22,11 +22,9 @@ import static javax.xml.xpath.XPathConstants.STRING; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -46,12 +44,11 @@ import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.TransformerFactoryConfigurationError; -import javax.xml.transform.sax.SAXSource; +import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; -import javax.xml.xpath.XPathFactoryConfigurationException; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.EventDriven; @@ -59,6 +56,9 @@ import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.SystemResourceConsiderations; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; @@ -76,16 +76,14 @@ import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.io.OutputStreamCallback; -import org.xml.sax.EntityResolver; -import org.xml.sax.InputSource; +import org.apache.nifi.processors.standard.xml.DocumentTypeAllowedDocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; +import org.w3c.dom.Document; -import net.sf.saxon.lib.NamespaceConstant; import net.sf.saxon.xpath.XPathEvaluator; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.XMLReaderFactory; +import net.sf.saxon.xpath.XPathFactoryImpl; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; @EventDriven @SideEffectFree @@ -105,6 +103,9 @@ import org.xml.sax.helpers.XMLReaderFactory; @WritesAttribute(attribute = "user-defined", description = "This processor adds user-defined attributes if the property is set to flowfile-attribute.") @DynamicProperty(name = "A FlowFile attribute(if is set to 'flowfile-attribute'", value = "An XPath expression", description = "If ='flowfile-attribute' " + "then the FlowFile attribute is set to the result of the XPath Expression. If ='flowfile-content' then the FlowFile content is set to the result of the XPath Expression.") +@SystemResourceConsiderations({ + @SystemResourceConsideration(resource = SystemResource.MEMORY, description = "Processing requires reading the entire FlowFile into memory") +}) public class EvaluateXPath extends AbstractProcessor { public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute"; @@ -137,7 +138,7 @@ public class EvaluateXPath extends AbstractProcessor { .description("Specifies whether or not the XML content should be validated against the DTD.") .required(true) .allowableValues("true", "false") - .defaultValue("true") + .defaultValue("false") .build(); public static final Relationship REL_MATCH = new Relationship.Builder() @@ -162,10 +163,6 @@ public class EvaluateXPath extends AbstractProcessor { private final AtomicReference factoryRef = new AtomicReference<>(); - static { - System.setProperty("javax.xml.xpath.XPathFactory:" + NamespaceConstant.OBJECT_MODEL_SAXON, "net.sf.saxon.xpath.XPathFactoryImpl"); - } - @Override protected void init(final ProcessorInitializationContext context) { final Set relationships = new HashSet<>(); @@ -215,8 +212,8 @@ public class EvaluateXPath extends AbstractProcessor { } @OnScheduled - public void initializeXPathFactory() throws XPathFactoryConfigurationException { - factoryRef.set(XPathFactory.newInstance(NamespaceConstant.OBJECT_MODEL_SAXON)); + public void initializeXPathFactory() { + factoryRef.set(new XPathFactoryImpl()); } @Override @@ -231,7 +228,6 @@ public class EvaluateXPath extends AbstractProcessor { } @Override - @SuppressWarnings("unchecked") public void onTrigger(final ProcessContext context, final ProcessSession session) { final List flowFiles = session.get(50); if (flowFiles.isEmpty()) { @@ -239,23 +235,6 @@ public class EvaluateXPath extends AbstractProcessor { } final ComponentLog logger = getLogger(); - final XMLReader xmlReader; - - try { - xmlReader = XMLReaderFactory.createXMLReader(); - } catch (SAXException e) { - logger.error("Error while constructing XMLReader {}", new Object[]{e}); - throw new ProcessException(e.getMessage()); - } - - if (!context.getProperty(VALIDATE_DTD).asBoolean()) { - xmlReader.setEntityResolver(new EntityResolver() { - @Override - public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException { - return new InputSource(new StringReader("")); - } - }); - } final XPathFactory factory = factoryRef.get(); final XPathEvaluator xpathEvaluator = (XPathEvaluator) factory.newXPath(); @@ -274,15 +253,6 @@ public class EvaluateXPath extends AbstractProcessor { } } - final XPathExpression slashExpression; - try { - slashExpression = xpathEvaluator.compile("/"); - } catch (XPathExpressionException e) { - logger.error("unable to compile XPath expression due to {}", new Object[]{e}); - session.transfer(flowFiles, REL_FAILURE); - return; - } - final String destination = context.getProperty(DESTINATION).getValue(); final QName returnType; @@ -306,27 +276,25 @@ public class EvaluateXPath extends AbstractProcessor { throw new IllegalStateException("There are no other return types..."); } + final boolean validatingDeclaration = context.getProperty(VALIDATE_DTD).asBoolean(); + flowFileLoop: for (FlowFile flowFile : flowFiles) { final AtomicReference error = new AtomicReference<>(null); final AtomicReference sourceRef = new AtomicReference<>(null); - session.read(flowFile, new InputStreamCallback() { - @Override - public void process(final InputStream rawIn) throws IOException { + try { + session.read(flowFile, rawIn -> { try (final InputStream in = new BufferedInputStream(rawIn)) { - final List rootList = (List) slashExpression.evaluate(new SAXSource(xmlReader, - new InputSource(in)), NODESET); - sourceRef.set(rootList.get(0)); - } catch (final Exception e) { - error.set(e); + final StandardDocumentProvider documentProvider = validatingDeclaration + ? new DocumentTypeAllowedDocumentProvider() + : new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); + sourceRef.set(new DOMSource(document)); } - } - }); - - if (error.get() != null) { - logger.error("unable to evaluate XPath against {} due to {}; routing to 'failure'", - new Object[]{flowFile, error.get()}); + }); + } catch (final Exception e) { + logger.error("Input parsing failed {}", flowFile, e); session.transfer(flowFile, REL_FAILURE); continue; } @@ -334,69 +302,60 @@ public class EvaluateXPath extends AbstractProcessor { final Map xpathResults = new HashMap<>(); for (final Map.Entry entry : attributeToXPathMap.entrySet()) { - Object result = null; + Object result; try { result = entry.getValue().evaluate(sourceRef.get(), returnType); if (result == null) { continue; } } catch (final XPathExpressionException e) { - logger.error("failed to evaluate XPath for {} for Property {} due to {}; routing to failure", - new Object[]{flowFile, entry.getKey(), e}); + logger.error("XPath Property [{}] evaluation on {} failed", flowFile, entry.getKey(), e); session.transfer(flowFile, REL_FAILURE); continue flowFileLoop; } if (returnType == NODESET) { - List nodeList = (List) result; - if (nodeList.isEmpty()) { - logger.info("Routing {} to 'unmatched'", new Object[]{flowFile}); + final NodeList nodeList = (NodeList) result; + if (nodeList.getLength() == 0) { + logger.info("XPath evaluation on {} produced no results", flowFile); session.transfer(flowFile, REL_NO_MATCH); continue flowFileLoop; - } else if (nodeList.size() > 1) { - logger.error("Routing {} to 'failure' because the XPath evaluated to {} XML nodes", - new Object[]{flowFile, nodeList.size()}); + } else if (nodeList.getLength() > 1) { + logger.error("XPath evaluation on {} produced unexpected results [{}]", flowFile, nodeList.getLength()); session.transfer(flowFile, REL_FAILURE); continue flowFileLoop; } - final Source sourceNode = nodeList.get(0); + final Node firstNode = nodeList.item(0); + final Source sourceNode = new DOMSource(firstNode); if (DESTINATION_ATTRIBUTE.equals(destination)) { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); doTransform(sourceNode, baos); - xpathResults.put(entry.getKey(), baos.toString("UTF-8")); - } catch (UnsupportedEncodingException e) { - throw new ProcessException(e); + xpathResults.put(entry.getKey(), new String(baos.toByteArray(), StandardCharsets.UTF_8)); } catch (TransformerException e) { error.set(e); } } else if (DESTINATION_CONTENT.equals(destination)) { - flowFile = session.write(flowFile, new OutputStreamCallback() { - @Override - public void process(final OutputStream rawOut) throws IOException { - try (final OutputStream out = new BufferedOutputStream(rawOut)) { - doTransform(sourceNode, out); - } catch (TransformerException e) { - error.set(e); - } + flowFile = session.write(flowFile, rawOut -> { + try (final OutputStream out = new BufferedOutputStream(rawOut)) { + doTransform(sourceNode, out); + } catch (TransformerException e) { + error.set(e); } }); } - } else if (returnType == STRING) { + } else { final String resultString = (String) result; if (DESTINATION_ATTRIBUTE.equals(destination)) { xpathResults.put(entry.getKey(), resultString); } else if (DESTINATION_CONTENT.equals(destination)) { - flowFile = session.write(flowFile, new OutputStreamCallback() { - @Override - public void process(final OutputStream rawOut) throws IOException { - try (final OutputStream out = new BufferedOutputStream(rawOut)) { - out.write(resultString.getBytes("UTF-8")); - } + flowFile = session.write(flowFile, rawOut -> { + try (final OutputStream out = new BufferedOutputStream(rawOut)) { + out.write(resultString.getBytes(StandardCharsets.UTF_8)); } }); } @@ -407,18 +366,16 @@ public class EvaluateXPath extends AbstractProcessor { if (DESTINATION_ATTRIBUTE.equals(destination)) { flowFile = session.putAllAttributes(flowFile, xpathResults); final Relationship destRel = xpathResults.isEmpty() ? REL_NO_MATCH : REL_MATCH; - logger.info("Successfully evaluated XPaths against {} and found {} matches; routing to {}", - new Object[]{flowFile, xpathResults.size(), destRel.getName()}); + logger.info("XPath evaluation on {} completed with results [{}]: content updated", flowFile, xpathResults.size()); session.transfer(flowFile, destRel); session.getProvenanceReporter().modifyAttributes(flowFile); } else if (DESTINATION_CONTENT.equals(destination)) { - logger.info("Successfully updated content for {}; routing to 'matched'", new Object[]{flowFile}); + logger.info("XPath evaluation on {} completed: content updated", flowFile); session.transfer(flowFile, REL_MATCH); session.getProvenanceReporter().modifyContent(flowFile); } } else { - logger.error("Failed to write XPath result for {} due to {}; routing original to 'failure'", - new Object[]{flowFile, error.get()}); + logger.error("XPath evaluation on {} failed", flowFile, error.get()); session.transfer(flowFile, REL_FAILURE); } } @@ -443,19 +400,19 @@ public class EvaluateXPath extends AbstractProcessor { final AtomicReference error = new AtomicReference<>(null); transformer.setErrorListener(new ErrorListener() { @Override - public void warning(final TransformerException exception) throws TransformerException { - logger.warn("Encountered warning from XPath Engine: ", new Object[]{exception.toString(), exception}); + public void warning(final TransformerException exception) { + logger.warn("Encountered warning from XPath Engine", exception); } @Override - public void error(final TransformerException exception) throws TransformerException { - logger.error("Encountered error from XPath Engine: ", new Object[]{exception.toString(), exception}); + public void error(final TransformerException exception) { + logger.error("Encountered error from XPath Engine", exception); error.set(exception); } @Override - public void fatalError(final TransformerException exception) throws TransformerException { - logger.error("Encountered warning from XPath Engine: ", new Object[]{exception.toString(), exception}); + public void fatalError(final TransformerException exception) { + logger.error("Encountered warning from XPath Engine", exception); error.set(exception); } }); @@ -471,7 +428,7 @@ public class EvaluateXPath extends AbstractProcessor { @Override public ValidationResult validate(final String subject, final String input, final ValidationContext validationContext) { try { - XPathFactory factory = XPathFactory.newInstance(NamespaceConstant.OBJECT_MODEL_SAXON); + XPathFactory factory = new XPathFactoryImpl(); final XPathEvaluator evaluator = (XPathEvaluator) factory.newXPath(); String error = null; @@ -484,7 +441,7 @@ public class EvaluateXPath extends AbstractProcessor { return new ValidationResult.Builder().input(input).subject(subject).valid(error == null).explanation(error).build(); } catch (final Exception e) { return new ValidationResult.Builder().input(input).subject(subject).valid(false) - .explanation("Unable to initialize XPath engine due to " + e.toString()).build(); + .explanation("Unable to initialize XPath engine due to " + e).build(); } } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java index cf58ab7aa1..7db4fdb280 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EvaluateXQuery.java @@ -22,7 +22,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; @@ -39,9 +38,8 @@ import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.TransformerFactoryConfigurationError; -import javax.xml.transform.sax.SAXSource; +import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import net.sf.saxon.s9api.DOMDestination; import net.sf.saxon.s9api.Processor; import net.sf.saxon.s9api.SaxonApiException; import net.sf.saxon.s9api.XQueryCompiler; @@ -56,6 +54,9 @@ import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.SystemResourceConsiderations; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; @@ -72,16 +73,10 @@ import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.io.OutputStreamCallback; import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.processors.standard.xml.DocumentTypeAllowedDocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; -import org.xml.sax.EntityResolver; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.XMLReaderFactory; @EventDriven @SideEffectFree @@ -103,6 +98,9 @@ import org.xml.sax.helpers.XMLReaderFactory; @WritesAttribute(attribute = "user-defined", description = "This processor adds user-defined attributes if the property is set to flowfile-attribute .") @DynamicProperty(name = "A FlowFile attribute(if is set to 'flowfile-attribute'", value = "An XQuery", description = "If ='flowfile-attribute' " + "then the FlowFile attribute is set to the result of the XQuery. If ='flowfile-content' then the FlowFile content is set to the result of the XQuery.") +@SystemResourceConsiderations({ + @SystemResourceConsideration(resource = SystemResource.MEMORY, description = "Processing requires reading the entire FlowFile into memory") +}) public class EvaluateXQuery extends AbstractProcessor { public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute"; @@ -112,8 +110,6 @@ public class EvaluateXQuery extends AbstractProcessor { public static final String OUTPUT_METHOD_HTML = "html"; public static final String OUTPUT_METHOD_TEXT = "text"; - public static final String UTF8 = "UTF-8"; - public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder() .name("Destination") .description( @@ -156,7 +152,7 @@ public class EvaluateXQuery extends AbstractProcessor { .description("Specifies whether or not the XML content should be validated against the DTD.") .required(true) .allowableValues("true", "false") - .defaultValue("true") + .defaultValue("false") .build(); public static final Relationship REL_MATCH = new Relationship.Builder() @@ -248,24 +244,6 @@ public class EvaluateXQuery extends AbstractProcessor { final Map attributeToXQueryMap = new HashMap<>(); final Processor proc = new Processor(false); - final XMLReader xmlReader; - - try { - xmlReader = XMLReaderFactory.createXMLReader(); - } catch (SAXException e) { - logger.error("Error while constructing XMLReader {}", new Object[]{e}); - throw new ProcessException(e.getMessage()); - } - - if (!context.getProperty(VALIDATE_DTD).asBoolean()) { - xmlReader.setEntityResolver(new EntityResolver() { - @Override - public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException { - return new InputSource(new StringReader("")); - } - }); - } - final XQueryCompiler comp = proc.newXQueryCompiler(); for (final Map.Entry entry : context.getProperties().entrySet()) { @@ -281,16 +259,8 @@ public class EvaluateXQuery extends AbstractProcessor { } } - final XQueryExecutable slashExpression; - try { - slashExpression = comp.compile("/"); - } catch (SaxonApiException e) { - logger.error("unable to compile XQuery expression due to {}", new Object[]{e}); - session.transfer(flowFileBatch, REL_FAILURE); - return; - } - final String destination = context.getProperty(DESTINATION).getValue(); + final boolean validateDeclaration = context.getProperty(VALIDATE_DTD).asBoolean(); flowFileLoop: for (FlowFile flowFile : flowFileBatch) { @@ -299,28 +269,20 @@ public class EvaluateXQuery extends AbstractProcessor { return; } - final AtomicReference error = new AtomicReference<>(null); - final AtomicReference sourceRef = new AtomicReference<>(null); - - session.read(flowFile, new InputStreamCallback() { - @Override - public void process(final InputStream rawIn) throws IOException { + final AtomicReference sourceRef = new AtomicReference<>(null); + try { + session.read(flowFile, rawIn -> { try (final InputStream in = new BufferedInputStream(rawIn)) { - XQueryEvaluator qe = slashExpression.load(); - qe.setSource(new SAXSource(xmlReader, new InputSource(in))); - Document dom = XmlUtils.createSafeDocumentBuilder(true).newDocument(); - qe.run(new DOMDestination(dom)); - XdmNode rootNode = proc.newDocumentBuilder().wrap(dom); - sourceRef.set(rootNode); - } catch (final Exception e) { - error.set(e); + final StandardDocumentProvider documentProvider = validateDeclaration + ? new DocumentTypeAllowedDocumentProvider() + : new StandardDocumentProvider(); + documentProvider.setNamespaceAware(true); + final Document document = documentProvider.parse(in); + sourceRef.set(new DOMSource(document)); } - } - }); - - if (error.get() != null) { - logger.error("unable to evaluate XQuery against {} due to {}; routing to 'failure'", - new Object[]{flowFile, error.get()}); + }); + } catch (final Exception e) { + logger.error("Input parsing failed {}", flowFile, e); session.transfer(flowFile, REL_FAILURE); continue; } @@ -331,7 +293,7 @@ public class EvaluateXQuery extends AbstractProcessor { for (final Map.Entry entry : attributeToXQueryMap.entrySet()) { try { XQueryEvaluator qe = entry.getValue().load(); - qe.setContextItem(sourceRef.get()); + qe.setSource(sourceRef.get()); XdmValue result = qe.evaluate(); if (DESTINATION_ATTRIBUTE.equals(destination)) { @@ -346,33 +308,27 @@ public class EvaluateXQuery extends AbstractProcessor { } } else { // if (DESTINATION_CONTENT.equals(destination)){ if (result.size() == 0) { - logger.info("Routing {} to 'unmatched'", new Object[]{flowFile}); + logger.info("No XQuery results found {}", flowFile); session.transfer(flowFile, REL_NO_MATCH); continue flowFileLoop; } else if (result.size() == 1) { final XdmItem item = result.itemAt(0); - flowFile = session.write(flowFile, new OutputStreamCallback() { - @Override - public void process(final OutputStream rawOut) throws IOException { - try (final OutputStream out = new BufferedOutputStream(rawOut)) { - writeformattedItem(item, context, out); - } catch (TransformerFactoryConfigurationError | TransformerException e) { - throw new IOException(e); - } + flowFile = session.write(flowFile, rawOut -> { + try (final OutputStream out = new BufferedOutputStream(rawOut)) { + writeformattedItem(item, context, out); + } catch (TransformerFactoryConfigurationError | TransformerException e) { + throw new IOException(e); } }); } else { for (final XdmItem item : result) { FlowFile ff = session.clone(flowFile); - ff = session.write(ff, new OutputStreamCallback() { - @Override - public void process(final OutputStream rawOut) throws IOException { - try (final OutputStream out = new BufferedOutputStream(rawOut)) { - try { - writeformattedItem(item, context, out); - } catch (TransformerFactoryConfigurationError | TransformerException e) { - throw new IOException(e); - } + ff = session.write(ff, rawOut -> { + try (final OutputStream out = new BufferedOutputStream(rawOut)) { + try { + writeformattedItem(item, context, out); + } catch (TransformerFactoryConfigurationError | TransformerException e) { + throw new IOException(e); } } }); @@ -381,14 +337,12 @@ public class EvaluateXQuery extends AbstractProcessor { } } } catch (final SaxonApiException e) { - logger.error("failed to evaluate XQuery for {} for Property {} due to {}; routing to failure", - new Object[]{flowFile, entry.getKey(), e}); + logger.error("XQuery Property [{}] processing failed", entry.getKey(), e); session.transfer(flowFile, REL_FAILURE); session.remove(childrenFlowFiles); continue flowFileLoop; } catch (TransformerFactoryConfigurationError | TransformerException | IOException e) { - logger.error("Failed to write XQuery result for {} due to {}; routing original to 'failure'", - new Object[]{flowFile, error.get()}); + logger.error("XQuery Property [{}] configuration failed", entry.getKey(), e); session.transfer(flowFile, REL_FAILURE); session.remove(childrenFlowFiles); continue flowFileLoop; @@ -398,18 +352,16 @@ public class EvaluateXQuery extends AbstractProcessor { if (DESTINATION_ATTRIBUTE.equals(destination)) { flowFile = session.putAllAttributes(flowFile, xQueryResults); final Relationship destRel = xQueryResults.isEmpty() ? REL_NO_MATCH : REL_MATCH; - logger.info("Successfully evaluated XQueries against {} and found {} matches; routing to {}", - new Object[]{flowFile, xQueryResults.size(), destRel.getName()}); + logger.info("XQuery results found [{}] for {}", xQueryResults.size(), flowFile); session.transfer(flowFile, destRel); session.getProvenanceReporter().modifyAttributes(flowFile); } else { // if (DESTINATION_CONTENT.equals(destination)) { if (!childrenFlowFiles.isEmpty()) { - logger.info("Successfully created {} new FlowFiles from {}; routing all to 'matched'", - new Object[]{childrenFlowFiles.size(), flowFile}); + logger.info("XQuery results found [{}] for {} FlowFiles created [{}]", xQueryResults.size(), flowFile, childrenFlowFiles.size()); session.transfer(childrenFlowFiles, REL_MATCH); session.remove(flowFile); } else { - logger.info("Successfully updated content for {}; routing to 'matched'", new Object[]{flowFile}); + logger.info("XQuery results found for {} content updated", flowFile); session.transfer(flowFile, REL_MATCH); session.getProvenanceReporter().modifyContent(flowFile); } @@ -475,7 +427,7 @@ public class EvaluateXQuery extends AbstractProcessor { return new ValidationResult.Builder().input(input).subject(subject).valid(error == null).explanation(error).build(); } catch (final Exception e) { return new ValidationResult.Builder().input(input).subject(subject).valid(false) - .explanation("Unable to initialize XQuery engine due to " + e.toString()).build(); + .explanation("Unable to initialize XQuery engine due to " + e).build(); } } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java index 889c5672a3..812b5c0a3a 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java @@ -23,6 +23,7 @@ import static org.apache.nifi.flowfile.attributes.FragmentAttributes.SEGMENT_ORI import static org.apache.nifi.flowfile.attributes.FragmentAttributes.copyAttributesToOriginal; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -34,8 +35,6 @@ import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.parsers.SAXParserFactory; import org.apache.commons.text.StringEscapeUtils; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.SystemResourceConsideration; @@ -59,15 +58,13 @@ import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.processors.standard.util.XmlElementNotifier; -import org.apache.nifi.security.xml.XmlUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.sax.StandardInputSourceParser; import org.xml.sax.Attributes; import org.xml.sax.ContentHandler; import org.xml.sax.InputSource; import org.xml.sax.Locator; import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; @EventDriven @SideEffectFree @@ -115,22 +112,6 @@ public class SplitXml extends AbstractProcessor { private List properties; private Set relationships; - private static final String FEATURE_PREFIX = "http://xml.org/sax/features/"; - public static final String ENABLE_NAMESPACES_FEATURE = FEATURE_PREFIX + "namespaces"; - public static final String ENABLE_NAMESPACE_PREFIXES_FEATURE = FEATURE_PREFIX + "namespace-prefixes"; - private static final SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); - - static { - saxParserFactory.setNamespaceAware(true); - try { - saxParserFactory.setFeature(ENABLE_NAMESPACES_FEATURE, true); - saxParserFactory.setFeature(ENABLE_NAMESPACE_PREFIXES_FEATURE, true); - } catch (Exception e) { - final Logger staticLogger = LoggerFactory.getLogger(SplitXml.class); - staticLogger.warn("Unable to configure SAX Parser to make namespaces available", e); - } - } - @Override protected void init(final ProcessorInitializationContext context) { final List properties = new ArrayList<>(); @@ -169,7 +150,7 @@ public class SplitXml extends AbstractProcessor { final AtomicInteger numberOfRecords = new AtomicInteger(0); final XmlSplitterSaxParser parser = new XmlSplitterSaxParser(xmlTree -> { FlowFile split = session.create(original); - split = session.write(split, out -> out.write(xmlTree.getBytes("UTF-8"))); + split = session.write(split, out -> out.write(xmlTree.getBytes(StandardCharsets.UTF_8))); split = session.putAttribute(split, FRAGMENT_ID.key(), fragmentIdentifier); split = session.putAttribute(split, FRAGMENT_INDEX.key(), Integer.toString(numberOfRecords.getAndIncrement())); split = session.putAttribute(split, SEGMENT_ORIGINAL_FILENAME.key(), split.getAttribute(CoreAttributes.FILENAME.key())); @@ -180,10 +161,11 @@ public class SplitXml extends AbstractProcessor { session.read(original, rawIn -> { try (final InputStream in = new java.io.BufferedInputStream(rawIn)) { try { - final XMLReader reader = XmlUtils.createSafeSaxReader(saxParserFactory, parser); - reader.parse(new InputSource(in)); - } catch (final ParserConfigurationException | SAXException e) { - logger.error("Unable to parse {} due to {}", new Object[]{original, e}); + final StandardInputSourceParser inputSourceParser = new StandardInputSourceParser(); + inputSourceParser.setNamespaceAware(true); + inputSourceParser.parse(new InputSource(in), parser); + } catch (final ProcessingException e) { + logger.error("Parsing failed {}", original, e); failed.set(true); } } @@ -200,7 +182,7 @@ public class SplitXml extends AbstractProcessor { final FlowFile originalToTransfer = copyAttributesToOriginal(session, original, fragmentIdentifier, numberOfRecords.get()); session.transfer(originalToTransfer, REL_ORIGINAL); - logger.info("Split {} into {} FlowFiles", new Object[]{originalToTransfer, splits.size()}); + logger.info("Split {} into {} FlowFiles", originalToTransfer, splits.size()); } } @@ -211,7 +193,7 @@ public class SplitXml extends AbstractProcessor { private final int splitDepth; private final StringBuilder sb = new StringBuilder(XML_PROLOGUE); private int depth = 0; - private Map prefixMap = new TreeMap<>(); + private final Map prefixMap = new TreeMap<>(); public XmlSplitterSaxParser(XmlElementNotifier notifier, int splitDepth) { this.notifier = notifier; @@ -252,11 +234,11 @@ public class SplitXml extends AbstractProcessor { } @Override - public void endDocument() throws SAXException { + public void endDocument() { } @Override - public void endElement(String uri, String localName, String qName) throws SAXException { + public void endElement(String uri, String localName, String qName) { // We have finished processing this element. Decrement the depth. int newDepth = --depth; @@ -279,16 +261,16 @@ public class SplitXml extends AbstractProcessor { } @Override - public void endPrefixMapping(String prefix) throws SAXException { + public void endPrefixMapping(String prefix) { prefixMap.remove(prefixToNamespace(prefix)); } @Override - public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException { + public void ignorableWhitespace(char[] ch, int start, int length) { } @Override - public void processingInstruction(String target, String data) throws SAXException { + public void processingInstruction(String target, String data) { } @Override @@ -296,15 +278,15 @@ public class SplitXml extends AbstractProcessor { } @Override - public void skippedEntity(String name) throws SAXException { + public void skippedEntity(String name) { } @Override - public void startDocument() throws SAXException { + public void startDocument() { } @Override - public void startElement(final String uri, final String localName, final String qName, final Attributes atts) throws SAXException { + public void startElement(final String uri, final String localName, final String qName, final Attributes atts) { // Increment the current depth because start a new XML element. int newDepth = ++depth; // Output the element and its attributes if it is @@ -343,7 +325,7 @@ public class SplitXml extends AbstractProcessor { } @Override - public void startPrefixMapping(String prefix, String uri) throws SAXException { + public void startPrefixMapping(String prefix, String uri) { final String ns = prefixToNamespace(prefix); prefixMap.put(ns, uri); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java index 49220a9e13..79a345ef8f 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java @@ -48,11 +48,12 @@ import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.StopWatch; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import javax.xml.XMLConstants; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.transform.OutputKeys; import javax.xml.transform.Result; @@ -363,10 +364,11 @@ public class TransformXml extends AbstractProcessor { } private Source getSecureSource(final StreamSource streamSource) throws TransformerConfigurationException { + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); try { - final XMLStreamReader streamReader = XmlUtils.createSafeReader(streamSource); + final XMLStreamReader streamReader = provider.getStreamReader(streamSource); return new StAXSource(streamReader); - } catch (final XMLStreamException e) { + } catch (final ProcessingException e) { throw new TransformerConfigurationException("XSLT Source Stream Reader creation failed", e); } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ValidateXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ValidateXml.java index b7184d4f06..0ae33391d8 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ValidateXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ValidateXml.java @@ -39,19 +39,22 @@ import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.io.InputStreamCallback; import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.security.xml.SafeXMLConfiguration; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; +import org.apache.nifi.xml.processing.validation.StandardSchemaValidator; +import org.apache.nifi.xml.processing.validation.SchemaValidator; import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.Source; +import javax.xml.transform.stax.StAXSource; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; -import javax.xml.validation.Validator; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -110,6 +113,10 @@ public class ValidateXml extends AbstractProcessor { private static final String SCHEMA_LANGUAGE = "http://www.w3.org/2001/XMLSchema"; + private static final SchemaValidator SCHEMA_VALIDATOR = new StandardSchemaValidator(); + + private static final XMLStreamReaderProvider READER_PROVIDER = new StandardXMLStreamReaderProvider(); + private List properties; private Set relationships; private final AtomicReference schemaRef = new AtomicReference<>(); @@ -156,41 +163,25 @@ public class ValidateXml extends AbstractProcessor { return; } - final Schema schema = schemaRef.get(); - final Validator validator = schema == null ? null : schema.newValidator(); final ComponentLog logger = getLogger(); final boolean attributeContainsXML = context.getProperty(XML_SOURCE_ATTRIBUTE).isSet(); for (FlowFile flowFile : flowFiles) { final AtomicBoolean valid = new AtomicBoolean(true); final AtomicReference exception = new AtomicReference<>(null); - SafeXMLConfiguration safeXMLConfiguration = new SafeXMLConfiguration(); - safeXMLConfiguration.setValidating(false); try { - DocumentBuilder docBuilder = safeXMLConfiguration.createDocumentBuilder(); - if (attributeContainsXML) { // If XML source attribute is set, validate attribute value String xml = flowFile.getAttribute(context.getProperty(XML_SOURCE_ATTRIBUTE).evaluateAttributeExpressions().getValue()); - ByteArrayInputStream bais = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); + ByteArrayInputStream inputStream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); - validate(validator, docBuilder, bais); + validate(inputStream); } else { // If XML source attribute is not set, validate flowfile content - session.read(flowFile, new InputStreamCallback() { - @Override - public void process(final InputStream in) throws IOException { - try { - validate(validator, docBuilder, in); - } catch (final IllegalArgumentException | SAXException e) { - valid.set(false); - exception.set(e); - } - } - }); + session.read(flowFile, inputStream -> validate(inputStream)); } - } catch (final IllegalArgumentException | SAXException | ParserConfigurationException | IOException e) { + } catch (final RuntimeException e) { valid.set(false); exception.set(e); } @@ -218,13 +209,23 @@ public class ValidateXml extends AbstractProcessor { } } - private void validate(final Validator validator, final DocumentBuilder docBuilder, final InputStream in) throws IllegalArgumentException, SAXException, IOException { - if (validator != null) { - // If schema is provided, validator will be non-null - validator.validate(new StreamSource(in)); + private void validate(final InputStream in) { + final Schema schema = schemaRef.get(); + if (schema == null) { + // Parse Document without schema validation + final XMLStreamReader reader = READER_PROVIDER.getStreamReader(new StreamSource(in)); + try { + while (reader.hasNext()) { + reader.next(); + } + } catch (final XMLStreamException e) { + throw new ProcessingException("Reading stream failed", e); + } } else { - // Only verify that the XML is well-formed; no schema check - docBuilder.parse(in); + final XMLStreamReaderProvider readerProvider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader reader = readerProvider.getStreamReader(new StreamSource(in)); + final Source source = new StAXSource(reader); + SCHEMA_VALIDATOR.validate(schema, source); } } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/DocumentReaderCallback.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/DocumentReaderCallback.java index 89d1485c61..f2230e2c3f 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/DocumentReaderCallback.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/DocumentReaderCallback.java @@ -18,12 +18,10 @@ package org.apache.nifi.processors.standard.util; import java.io.IOException; import java.io.InputStream; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; -import org.xml.sax.SAXException; public class DocumentReaderCallback implements InputStreamCallback { @@ -42,10 +40,11 @@ public class DocumentReaderCallback implements InputStreamCallback { @Override public void process(final InputStream stream) throws IOException { try { - DocumentBuilder builder = XmlUtils.createSafeDocumentBuilder(isNamespaceAware); - document = builder.parse(stream); - } catch (ParserConfigurationException | SAXException pce) { - throw new IOException(pce.getLocalizedMessage(), pce); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + documentProvider.setNamespaceAware(isNamespaceAware); + document = documentProvider.parse(stream); + } catch (final ProcessingException e) { + throw new IOException(e.getLocalizedMessage(), e); } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/xml/DocumentTypeAllowedDocumentProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/xml/DocumentTypeAllowedDocumentProvider.java new file mode 100644 index 0000000000..be54f7a701 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/xml/DocumentTypeAllowedDocumentProvider.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.standard.xml; + +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; + +/** + * Document Provider implementation that allows local Document Type Declarations + */ +public class DocumentTypeAllowedDocumentProvider extends StandardDocumentProvider { + /** + * Enable Document Type Declaration through disabling disallow configuration + * + * @return Disallow Disabled + */ + @Override + protected boolean isDisallowDocumentTypeDeclaration() { + return false; + } +} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXPath.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXPath.java index 98899bbe14..95d366a859 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXPath.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXPath.java @@ -16,19 +16,19 @@ */ package org.apache.nifi.processors.standard; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import javax.xml.xpath.XPathFactoryConfigurationException; - import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestEvaluateXPath { @@ -37,7 +37,7 @@ public class TestEvaluateXPath { private static final Path XML_SNIPPET_NONEXISTENT_DOCTYPE = Paths.get("src/test/resources/TestXml/xml-snippet-external-doctype.xml"); @Test - public void testAsAttribute() throws XPathFactoryConfigurationException, IOException { + public void testAsAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_ATTRIBUTE); testRunner.setProperty("xpath.result1", "/"); @@ -53,7 +53,7 @@ public class TestEvaluateXPath { } @Test - public void testCheckIfElementExists() throws XPathFactoryConfigurationException, IOException { + public void testCheckIfElementExists() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_ATTRIBUTE); testRunner.setProperty("xpath.result1", "/"); @@ -71,7 +71,7 @@ public class TestEvaluateXPath { } @Test - public void testUnmatched() throws XPathFactoryConfigurationException, IOException { + public void testUnmatched() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty("xpath.result.exist.2", "/*:bundle/node2"); @@ -83,7 +83,7 @@ public class TestEvaluateXPath { testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_NO_MATCH).get(0).assertContentEquals(XML_SNIPPET); } - @Test(expected = java.lang.AssertionError.class) + @Test public void testMultipleXPathForContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); @@ -92,11 +92,12 @@ public class TestEvaluateXPath { testRunner.setProperty("some.property.2", "/*:bundle/node/subNode[2]"); testRunner.enqueue(XML_SNIPPET); - testRunner.run(); + + assertThrows(AssertionError.class, testRunner::run); } @Test - public void testWriteToContent() throws XPathFactoryConfigurationException, IOException { + public void testWriteToContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty("some.property", "/*:bundle/node/subNode[1]"); @@ -107,13 +108,13 @@ public class TestEvaluateXPath { testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0); final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); + final String outXml = new String(outData, StandardCharsets.UTF_8); assertTrue(outXml.contains("subNode")); assertTrue(outXml.contains("Hello")); } @Test - public void testFailureIfContentMatchesMultipleNodes() throws XPathFactoryConfigurationException, IOException { + public void testFailureIfContentMatchesMultipleNodes() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty("some.property", "/*:bundle/node/subNode"); @@ -125,7 +126,7 @@ public class TestEvaluateXPath { } @Test - public void testWriteStringToContent() throws XPathFactoryConfigurationException, IOException { + public void testWriteStringToContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING); @@ -136,13 +137,11 @@ public class TestEvaluateXPath { testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("Hello")); + out.assertContentEquals("Hello"); } @Test - public void testWriteNodeSetToAttribute() throws XPathFactoryConfigurationException, IOException { + public void testWriteNodeSetToAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_ATTRIBUTE); testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_NODESET); @@ -159,7 +158,7 @@ public class TestEvaluateXPath { } @Test - public void testSuccessForEmbeddedDocTypeValidation() throws XPathFactoryConfigurationException, IOException { + public void testSuccessForEmbeddedDocTypeValidation() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING); @@ -171,13 +170,11 @@ public class TestEvaluateXPath { testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("Hello")); + out.assertContentEquals("Hello"); } @Test - public void testSuccessForEmbeddedDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException { + public void testFailureForEmbeddedDocTypeValidationDisabled() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING); @@ -187,15 +184,11 @@ public class TestEvaluateXPath { testRunner.enqueue(XML_SNIPPET_EMBEDDED_DOCTYPE); testRunner.run(); - testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("Hello")); + testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_FAILURE, 1); } @Test - public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws XPathFactoryConfigurationException, IOException { + public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING); @@ -208,7 +201,7 @@ public class TestEvaluateXPath { } @Test - public void testSuccessForExternalDocTypeWithDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException { + public void testFailureForExternalDocTypeWithDocTypeValidationDisabled() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXPath()); testRunner.setProperty(EvaluateXPath.DESTINATION, EvaluateXPath.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXPath.RETURN_TYPE, EvaluateXPath.RETURN_TYPE_STRING); @@ -218,11 +211,6 @@ public class TestEvaluateXPath { testRunner.enqueue(XML_SNIPPET_NONEXISTENT_DOCTYPE); testRunner.run(); - testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_MATCH, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXPath.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("Hello")); + testRunner.assertAllFlowFilesTransferred(EvaluateXPath.REL_FAILURE, 1); } - } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXQuery.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXQuery.java index 05fe9c278f..7282f07244 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXQuery.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEvaluateXQuery.java @@ -16,15 +16,18 @@ */ package org.apache.nifi.processors.standard; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,13 +35,12 @@ import java.util.Map.Entry; import java.util.Properties; import javax.xml.transform.OutputKeys; -import javax.xml.xpath.XPathFactoryConfigurationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestEvaluateXQuery { @@ -52,16 +54,15 @@ public class TestEvaluateXQuery { private static final boolean[] booleans = {true, false}; @Test - public void testSetTransformerProperties() throws Exception { - - for (int i = 0; i < methods.length; i++) { - for (int j = 0; j < booleans.length; j++) { - for (int k = 0; k < booleans.length; k++) { - Properties props = EvaluateXQuery.getTransformerProperties(methods[i], booleans[j], booleans[k]); + public void testSetTransformerProperties() { + for (final String method : methods) { + for (final boolean indent : booleans) { + for (final boolean omitDeclaration : booleans) { + Properties props = EvaluateXQuery.getTransformerProperties(method, indent, omitDeclaration); assertEquals(3, props.size()); - assertEquals(methods[i], props.getProperty(OutputKeys.METHOD)); - assertEquals(booleans[j] ? "yes" : "no", props.getProperty(OutputKeys.INDENT)); - assertEquals(booleans[k] ? "yes" : "no", props.getProperty(OutputKeys.OMIT_XML_DECLARATION)); + assertEquals(method, props.getProperty(OutputKeys.METHOD)); + assertEquals(indent ? "yes" : "no", props.getProperty(OutputKeys.INDENT)); + assertEquals(omitDeclaration ? "yes" : "no", props.getProperty(OutputKeys.OMIT_XML_DECLARATION)); } } } @@ -75,20 +76,20 @@ public class TestEvaluateXQuery { final String singleElementNodeQuery = "//fruit[1]"; final String singleTextNodeQuery = "//fruit[1]/name/text()"; - for (int i = 0; i < methods.length; i++) { - for (int j = 0; j < booleans.length; j++) { - for (int k = 0; k < booleans.length; k++) { - formattedResults = getFormattedResult(XML_SNIPPET, atomicQuery, methods[i], booleans[j], booleans[k]); + for (final String method : methods) { + for (final boolean indent : booleans) { + for (final boolean omitDeclaration : booleans) { + formattedResults = getFormattedResult(XML_SNIPPET, atomicQuery, method, indent, omitDeclaration); assertEquals(1, formattedResults.size()); assertEquals("7", formattedResults.get(0)); } } } - for (int i = 0; i < methods.length; i++) { - for (int j = 0; j < booleans.length; j++) { - for (int k = 0; k < booleans.length; k++) { - formattedResults = getFormattedResult(XML_SNIPPET, singleTextNodeQuery, methods[i], booleans[j], booleans[k]); + for (final String method : methods) { + for (final boolean indent : booleans) { + for (final boolean omitDeclaration : booleans) { + formattedResults = getFormattedResult(XML_SNIPPET, singleTextNodeQuery, method, indent, omitDeclaration); assertEquals(1, formattedResults.size()); assertEquals("apple", formattedResults.get(0)); } @@ -161,7 +162,6 @@ public class TestEvaluateXQuery { List resultFlowFiles; List resultStrings = new ArrayList<>(); - runnerProps.clear(); runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_CONTENT); runnerProps.put(EvaluateXQuery.XML_OUTPUT_METHOD.getName(), method); runnerProps.put(EvaluateXQuery.XML_OUTPUT_INDENT.getName(), Boolean.toString(indent)); @@ -169,45 +169,44 @@ public class TestEvaluateXQuery { runnerProps.put("xquery", xQuery); resultFlowFiles = runXquery(xml, runnerProps); - for (int i = 0; i < resultFlowFiles.size(); i++) { - final MockFlowFile out = resultFlowFiles.get(i); - final byte[] outData = out.toByteArray(); - final String outXml = new String(outData, "UTF-8"); + for (final MockFlowFile flowFile : resultFlowFiles) { + final byte[] outData = flowFile.toByteArray(); + final String outXml = new String(outData, StandardCharsets.UTF_8); resultStrings.add(outXml); } return resultStrings; } - @Test(expected = java.lang.AssertionError.class) - public void testBadXQuery() throws Exception { - doXqueryTest(XML_SNIPPET, "counttttttt(*:fruitbasket/fruit)", Arrays.asList("7")); + @Test + public void testBadXQuery() { + assertThrows(AssertionError.class, () -> doXqueryTest(XML_SNIPPET, "counttttttt(*:fruitbasket/fruit)", Collections.singletonList("7"))); } @Test public void testXQueries() throws Exception { /* count matches */ - doXqueryTest(XML_SNIPPET, "count(*:fruitbasket/fruit)", Arrays.asList("7")); - doXqueryTest(XML_SNIPPET, "count(//fruit)", Arrays.asList("7")); + doXqueryTest(XML_SNIPPET, "count(*:fruitbasket/fruit)", Collections.singletonList("7")); + doXqueryTest(XML_SNIPPET, "count(//fruit)", Collections.singletonList("7")); /* Using a namespace */ - doXqueryTest(XML_SNIPPET, "declare namespace fb = \"http://namespace/1\"; count(fb:fruitbasket/fruit)", Arrays.asList("7")); + doXqueryTest(XML_SNIPPET, "declare namespace fb = \"http://namespace/1\"; count(fb:fruitbasket/fruit)", Collections.singletonList("7")); /* determine if node exists */ - doXqueryTest(XML_SNIPPET, "boolean(//fruit[1])", Arrays.asList("true")); - doXqueryTest(XML_SNIPPET, "boolean(//fruit[100])", Arrays.asList("false")); + doXqueryTest(XML_SNIPPET, "boolean(//fruit[1])", Collections.singletonList("true")); + doXqueryTest(XML_SNIPPET, "boolean(//fruit[100])", Collections.singletonList("false")); /* XML first match */ - doXqueryTest(XML_SNIPPET, "//fruit[1]", Arrays.asList( + doXqueryTest(XML_SNIPPET, "//fruit[1]", Collections.singletonList( "applered")); /* XML last match */ - doXqueryTest(XML_SNIPPET, "//fruit[count(//fruit)]", Arrays.asList( + doXqueryTest(XML_SNIPPET, "//fruit[count(//fruit)]", Collections.singletonList( "none")); /* XML first match wrapped */ - doXqueryTest(XML_SNIPPET, "{//fruit[1]}", Arrays.asList( + doXqueryTest(XML_SNIPPET, "{//fruit[1]}", Collections.singletonList( "applered")); /* XML all matches (multiple results) */ @@ -221,7 +220,7 @@ public class TestEvaluateXQuery { "none")); /* XML all matches wrapped (one result)*/ - doXqueryTest(XML_SNIPPET, "{//fruit}", Arrays.asList( + doXqueryTest(XML_SNIPPET, "{//fruit}", Collections.singletonList( "" + "" + "applered" @@ -237,18 +236,18 @@ public class TestEvaluateXQuery { doXqueryTest(XML_SNIPPET, "for $x in //fruit return $x/name/text()", Arrays.asList(fruitNames)); /* String first match fruit name (XPath)*/ - doXqueryTest(XML_SNIPPET, "//fruit[1]/name/text()", Arrays.asList("apple")); + doXqueryTest(XML_SNIPPET, "//fruit[1]/name/text()", Collections.singletonList("apple")); /* String first match fruit color (XPath)*/ - doXqueryTest(XML_SNIPPET, "//fruit[1]/color/text()", Arrays.asList("red")); + doXqueryTest(XML_SNIPPET, "//fruit[1]/color/text()", Collections.singletonList("red")); /* String first match fruit name (XQuery)*/ doXqueryTest(XML_SNIPPET, "for $x in //fruit[1] return string-join(($x/name/text() , $x/color/text()), ' - ')", - Arrays.asList("apple - red")); + Collections.singletonList("apple - red")); /* String first match fruit & color (one result)*/ doXqueryTest(XML_SNIPPET, "for $x in //fruit[1] return string-join(($x/name/text() , $x/color/text()), ' - ')", - Arrays.asList("apple - red")); + Collections.singletonList("apple - red")); /* String all matches fruit & color (multiple results)*/ doXqueryTest(XML_SNIPPET, "for $x in //fruit return string-join(($x/name/text() , $x/color/text()), ' - ')", @@ -263,7 +262,7 @@ public class TestEvaluateXQuery { /* String all matches fruit & color (single, newline delimited result)*/ doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit return string-join(($x/name/text() , $x/color/text()), ' - ')) return $y), '\n')", - Arrays.asList( + Collections.singletonList( "apple - red\n" + "apple - green\n" + "banana - yellow\n" @@ -274,7 +273,7 @@ public class TestEvaluateXQuery { /* String all matches fruit & color using "let" (single, newline delimited result)*/ doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit let $d := string-join(($x/name/text() , $x/color/text()), ' - ') return $d) return $y), '\n')", - Arrays.asList( + Collections.singletonList( "apple - red\n" + "apple - green\n" + "banana - yellow\n" @@ -285,25 +284,25 @@ public class TestEvaluateXQuery { /* String all matches name only, comma delimited (one result)*/ doXqueryTest(XML_SNIPPET, "string-join((for $x in //fruit return $x/name/text()), ', ')", - Arrays.asList("apple, apple, banana, orange, blueberry, raspberry, none")); + Collections.singletonList("apple, apple, banana, orange, blueberry, raspberry, none")); /* String all matches color and name, comma delimited (one result)*/ doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit return string-join(($x/color/text() , $x/name/text()), ' ')) return $y), ', ')", - Arrays.asList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none")); + Collections.singletonList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none")); /* String all matches color and name, comma delimited using let(one result)*/ doXqueryTest(XML_SNIPPET, "string-join((for $y in (for $x in //fruit let $d := string-join(($x/color/text() , $x/name/text()), ' ') return $d) return $y), ', ')", - Arrays.asList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none")); + Collections.singletonList("red apple, green apple, yellow banana, orange orange, blue blueberry, red raspberry, none")); /* Query for attribute */ - doXqueryTest(XML_SNIPPET, "string(//fruit[1]/@taste)", Arrays.asList("crisp")); + doXqueryTest(XML_SNIPPET, "string(//fruit[1]/@taste)", Collections.singletonList("crisp")); /* Query for comment */ - doXqueryTest(XML_SNIPPET, "//fruit/comment()", Arrays.asList(" Apples are my favorite ")); + doXqueryTest(XML_SNIPPET, "//fruit/comment()", Collections.singletonList(" Apples are my favorite ")); /* Query for processing instruction */ - doXqueryTest(XML_SNIPPET, "//processing-instruction()[name()='xml-stylesheet']", Arrays.asList("type=\"text/xsl\" href=\"foo.xsl\"")); + doXqueryTest(XML_SNIPPET, "//processing-instruction()[name()='xml-stylesheet']", Collections.singletonList("type=\"text/xsl\" href=\"foo.xsl\"")); } @@ -313,50 +312,44 @@ public class TestEvaluateXQuery { List resultFlowFiles; // test read from content, write to attribute - { - runnerProps.clear(); - runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_ATTRIBUTE); - runnerProps.put("xquery", xQuery); - resultFlowFiles = runXquery(xml, runnerProps); + runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_ATTRIBUTE); + runnerProps.put("xquery", xQuery); + resultFlowFiles = runXquery(xml, runnerProps); - assertEquals(1, resultFlowFiles.size()); + assertEquals(1, resultFlowFiles.size()); - final MockFlowFile out = resultFlowFiles.get(0); + final MockFlowFile out = resultFlowFiles.get(0); - for (int i = 0; i < expectedResults.size(); i++) { - String key = "xquery"; - if (expectedResults.size() > 1) { - key += "." + ((int) i + 1); - } - final String actual = out.getAttribute(key).replaceAll(">\\s+<", "><"); - final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><"); - assertEquals(expected, actual); + for (int i = 0; i < expectedResults.size(); i++) { + String key = "xquery"; + if (expectedResults.size() > 1) { + key += "." + (i + 1); } + final String actual = out.getAttribute(key).replaceAll(">\\s+<", "><"); + final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><"); + assertEquals(expected, actual); } // test read from content, write to content - { - runnerProps.clear(); - runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_CONTENT); - runnerProps.put("xquery", xQuery); - resultFlowFiles = runXquery(xml, runnerProps); + runnerProps.clear(); + runnerProps.put(EvaluateXQuery.DESTINATION.getName(), EvaluateXQuery.DESTINATION_CONTENT); + runnerProps.put("xquery", xQuery); + resultFlowFiles = runXquery(xml, runnerProps); - assertEquals(expectedResults.size(), resultFlowFiles.size()); + assertEquals(expectedResults.size(), resultFlowFiles.size()); - for (int i = 0; i < resultFlowFiles.size(); i++) { + for (int i = 0; i < resultFlowFiles.size(); i++) { - final MockFlowFile out = resultFlowFiles.get(i); - final byte[] outData = out.toByteArray(); - final String outXml = new String(outData, "UTF-8").replaceAll(">\\s+<", "><"); - final String actual = outXml; - final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><"); - assertEquals(expected, actual); - } + final MockFlowFile resultFlowFile = resultFlowFiles.get(i); + final byte[] outData = resultFlowFile.toByteArray(); + final String outXml = new String(outData, StandardCharsets.UTF_8).replaceAll(">\\s+<", "><"); + final String expected = expectedResults.get(i).replaceAll(">\\s+<", "><"); + assertEquals(expected, outXml); } } private List runXquery(Path xml, Map runnerProps) throws Exception { - return runXquery(xml, runnerProps, new HashMap()); + return runXquery(xml, runnerProps, new HashMap<>()); } private List runXquery(Path xml, Map runnerProps, Map flowFileAttributes) throws Exception { @@ -376,7 +369,7 @@ public class TestEvaluateXQuery { } @Test - public void testRootPath() throws XPathFactoryConfigurationException, IOException { + public void testRootPath() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("xquery.result1", "/"); @@ -387,13 +380,13 @@ public class TestEvaluateXQuery { testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); final String attributeString = out.getAttribute("xquery.result1").replaceAll(">\\s+<", "><"); - final String xmlSnippetString = new String(Files.readAllBytes(XML_SNIPPET), "UTF-8").replaceAll(">\\s+<", "><"); + final String xmlSnippetString = new String(Files.readAllBytes(XML_SNIPPET), StandardCharsets.UTF_8).replaceAll(">\\s+<", "><"); assertEquals(xmlSnippetString, attributeString); } @Test - public void testCheckIfElementExists() throws XPathFactoryConfigurationException, IOException { + public void testCheckIfElementExists() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("xquery.result.exist.1", "boolean(/*:fruitbasket/fruit[1])"); @@ -409,7 +402,7 @@ public class TestEvaluateXQuery { } @Test - public void testUnmatchedContent() throws XPathFactoryConfigurationException, IOException { + public void testUnmatchedContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty("xquery.result.exist.2", "/*:fruitbasket/node2"); @@ -422,7 +415,7 @@ public class TestEvaluateXQuery { } @Test - public void testUnmatchedAttribute() throws XPathFactoryConfigurationException, IOException { + public void testUnmatchedAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("xquery.result.exist.2", "/*:fruitbasket/node2"); @@ -437,7 +430,7 @@ public class TestEvaluateXQuery { } @Test - public void testNoXQueryAttribute() throws XPathFactoryConfigurationException, IOException { + public void testNoXQueryAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); @@ -448,17 +441,17 @@ public class TestEvaluateXQuery { testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_NO_MATCH).get(0).assertContentEquals(XML_SNIPPET); } - @Test(expected = java.lang.AssertionError.class) - public void testNoXQueryContent() throws XPathFactoryConfigurationException, IOException { + @Test + public void testNoXQueryContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.enqueue(XML_SNIPPET); - testRunner.run(); + assertThrows(AssertionError.class, testRunner::run); } @Test - public void testOneMatchOneUnmatchAttribute() throws XPathFactoryConfigurationException, IOException { + public void testOneMatchOneUnmatchAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("some.property", "//fruit/name/text()"); @@ -472,7 +465,7 @@ public class TestEvaluateXQuery { final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); for (int i = 0; i < fruitNames.length; i++) { - final String outXml = out.getAttribute("some.property." + ((int) i + 1)); + final String outXml = out.getAttribute("some.property." + (i + 1)); assertEquals(fruitNames[i], outXml.trim()); } @@ -481,7 +474,7 @@ public class TestEvaluateXQuery { } @Test - public void testMatchedEmptyStringAttribute() throws XPathFactoryConfigurationException, IOException { + public void testMatchedEmptyStringAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("xquery.result.exist.2", "/*:fruitbasket/*[name='none']/color/text()"); @@ -496,7 +489,7 @@ public class TestEvaluateXQuery { testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_NO_MATCH).get(0).assertContentEquals(XML_SNIPPET); } - @Test(expected = java.lang.AssertionError.class) + @Test public void testMultipleXPathForContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); @@ -504,11 +497,11 @@ public class TestEvaluateXQuery { testRunner.setProperty("some.property.2", "/*:fruitbasket/fruit[2]"); testRunner.enqueue(XML_SNIPPET); - testRunner.run(); + assertThrows(AssertionError.class, testRunner::run); } @Test - public void testWriteStringToAttribute() throws XPathFactoryConfigurationException, IOException { + public void testWriteStringToAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("xquery.result2", "/*:fruitbasket/fruit[1]/name/text()"); @@ -523,7 +516,7 @@ public class TestEvaluateXQuery { } @Test - public void testWriteStringToContent() throws XPathFactoryConfigurationException, IOException { + public void testWriteStringToContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name/text()"); @@ -533,13 +526,11 @@ public class TestEvaluateXQuery { testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("apple")); + out.assertContentEquals("apple"); } @Test - public void testWriteXmlToAttribute() throws XPathFactoryConfigurationException, IOException { + public void testWriteXmlToAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name"); @@ -555,7 +546,7 @@ public class TestEvaluateXQuery { } @Test - public void testWriteXmlToContent() throws XPathFactoryConfigurationException, IOException { + public void testWriteXmlToContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty("some.property", "/*:fruitbasket/fruit[1]/name"); @@ -566,12 +557,12 @@ public class TestEvaluateXQuery { testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); + final String outXml = new String(outData, StandardCharsets.UTF_8); assertTrue(outXml.contains("apple")); } @Test - public void testMatchesMultipleStringContent() throws XPathFactoryConfigurationException, IOException { + public void testMatchesMultipleStringContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty("some.property", "//fruit/name/text()"); @@ -585,14 +576,12 @@ public class TestEvaluateXQuery { for (int i = 0; i < flowFilesForRelMatch.size(); i++) { final MockFlowFile out = flowFilesForRelMatch.get(i); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertEquals(fruitNames[i], outXml.trim()); + out.assertContentEquals(fruitNames[i]); } } @Test - public void testMatchesMultipleStringAttribute() throws XPathFactoryConfigurationException, IOException { + public void testMatchesMultipleStringAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("some.property", "//fruit/name/text()"); @@ -605,14 +594,14 @@ public class TestEvaluateXQuery { final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); for (int i = 0; i < fruitNames.length; i++) { - final String outXml = out.getAttribute("some.property." + ((int) i + 1)); + final String outXml = out.getAttribute("some.property." + (i + 1)); assertEquals(fruitNames[i], outXml.trim()); } testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0).assertContentEquals(XML_SNIPPET); } @Test - public void testMatchesMultipleXmlContent() throws XPathFactoryConfigurationException, IOException { + public void testMatchesMultipleXmlContent() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty("some.property", "//fruit/name"); @@ -627,14 +616,14 @@ public class TestEvaluateXQuery { final MockFlowFile out = flowFilesForRelMatch.get(i); final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); + final String outXml = new String(outData, StandardCharsets.UTF_8); String expectedXml = "" + fruitNames[i] + ""; assertEquals(expectedXml, outXml.trim()); } } @Test - public void testMatchesMultipleXmlAttribute() throws XPathFactoryConfigurationException, IOException { + public void testMatchesMultipleXmlAttribute() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_ATTRIBUTE); testRunner.setProperty("some.property", "//fruit/name"); @@ -647,7 +636,7 @@ public class TestEvaluateXQuery { final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); for (int i = 0; i < fruitNames.length; i++) { - final String outXml = out.getAttribute("some.property." + ((int) i + 1)); + final String outXml = out.getAttribute("some.property." + (i + 1)); String expectedXml = "" + fruitNames[i] + ""; assertEquals(expectedXml, outXml.trim()); } @@ -655,7 +644,7 @@ public class TestEvaluateXQuery { } @Test - public void testSuccessForEmbeddedDocTypeValidation() throws XPathFactoryConfigurationException, IOException { + public void testSuccessForEmbeddedDocTypeValidation() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXQuery.VALIDATE_DTD, "true"); @@ -666,13 +655,11 @@ public class TestEvaluateXQuery { testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("Hello")); + out.assertContentEquals("Hello"); } @Test - public void testSuccessForEmbeddedDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException { + public void testFailureForEmbeddedDocTypeValidationDisabled() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXQuery.VALIDATE_DTD, "false"); @@ -681,16 +668,12 @@ public class TestEvaluateXQuery { testRunner.enqueue(XML_SNIPPET_EMBEDDED_DOCTYPE); testRunner.run(); - testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("Hello")); + testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_FAILURE, 1); } @Test - public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws XPathFactoryConfigurationException, IOException { + public void testFailureForExternalDocTypeWithDocTypeValidationEnabled() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty("some.property", "/*:bundle/node/subNode[1]/value/text()"); @@ -701,9 +684,8 @@ public class TestEvaluateXQuery { testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_FAILURE, 1); } - @Test - public void testSuccessForExternalDocTypeWithDocTypeValidationDisabled() throws XPathFactoryConfigurationException, IOException { + public void testFailureForExternalDocTypeWithDocTypeValidationDisabled() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EvaluateXQuery()); testRunner.setProperty(EvaluateXQuery.DESTINATION, EvaluateXQuery.DESTINATION_CONTENT); testRunner.setProperty(EvaluateXQuery.VALIDATE_DTD, "false"); @@ -712,10 +694,6 @@ public class TestEvaluateXQuery { testRunner.enqueue(XML_SNIPPET_NONEXISTENT_DOCTYPE); testRunner.run(); - testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_MATCH, 1); - final MockFlowFile out = testRunner.getFlowFilesForRelationship(EvaluateXQuery.REL_MATCH).get(0); - final byte[] outData = testRunner.getContentAsByteArray(out); - final String outXml = new String(outData, "UTF-8"); - assertTrue(outXml.trim().equals("Hello")); + testRunner.assertAllFlowFilesTransferred(EvaluateXQuery.REL_FAILURE, 1); } } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index d30200e511..dd72dd83a9 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -176,11 +176,6 @@ ${nifi.groovy.version} test - - org.apache.nifi - nifi-security-utils - 1.17.0-SNAPSHOT - diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/XMLFileLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/XMLFileLookupService.java index ce9c37763d..aaeb8f21c9 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/XMLFileLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/XMLFileLookupService.java @@ -19,7 +19,7 @@ package org.apache.nifi.lookup; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.lookup.configuration2.CommonsConfigurationLookupService; -import org.apache.nifi.security.xml.SafeXMLConfiguration; +import org.apache.nifi.lookup.configuration2.SafeXMLConfiguration; @Tags({"lookup", "cache", "enrich", "join", "xml", "reloadable", "key", "value"}) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/CommonsConfigurationLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/CommonsConfigurationLookupService.java index cbc17f4b62..6263a77cd0 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/CommonsConfigurationLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/CommonsConfigurationLookupService.java @@ -36,7 +36,6 @@ import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.lookup.LookupFailureException; import org.apache.nifi.lookup.StringLookupService; import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.security.xml.XXEValidator; import java.io.File; import java.lang.reflect.ParameterizedType; diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/SafeXMLConfiguration.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/SafeXMLConfiguration.java similarity index 97% rename from nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/SafeXMLConfiguration.java rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/SafeXMLConfiguration.java index 44f29dcb40..5de11266b6 100644 --- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/SafeXMLConfiguration.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/SafeXMLConfiguration.java @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.security.xml; +package org.apache.nifi.lookup.configuration2; import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -45,7 +46,7 @@ public class SafeXMLConfiguration extends XMLConfiguration { private static final String W3C_XML_SCHEMA = "http://www.w3.org/2001/XMLSchema"; - // These features are used to disable processing external entities in the DocumentBuilderFactory to protect against XXE attacks + // These features are used to disable processing external entities to protect against XXE attacks private static final String DISALLOW_DOCTYPES = "http://apache.org/xml/features/disallow-doctype-decl"; private static final String ALLOW_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities"; private static final String ALLOW_EXTERNAL_PARAM_ENTITIES = "http://xml.org/sax/features/external-parameter-entities"; @@ -90,6 +91,7 @@ public class SafeXMLConfiguration extends XMLConfiguration { } // Disable DTDs and external entities to protect against XXE + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); factory.setAttribute(DISALLOW_DOCTYPES, true); factory.setAttribute(ALLOW_EXTERNAL_GENERAL_ENTITIES, false); factory.setAttribute(ALLOW_EXTERNAL_PARAM_ENTITIES, false); diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XXEValidator.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/XXEValidator.java similarity index 99% rename from nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XXEValidator.java rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/XXEValidator.java index e9c54d5f41..087e7cb2ff 100644 --- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XXEValidator.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/configuration2/XXEValidator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.security.xml; +package org.apache.nifi.lookup.configuration2; import java.io.BufferedReader; import java.io.IOException; diff --git a/nifi-commons/nifi-security-utils/src/test/java/org/apache/nifi/security/XXEValidatorTest.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/java/org/apache/nifi/lookup/configuration2/XXEValidatorTest.java similarity index 98% rename from nifi-commons/nifi-security-utils/src/test/java/org/apache/nifi/security/XXEValidatorTest.java rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/java/org/apache/nifi/lookup/configuration2/XXEValidatorTest.java index fd2dceac38..a8aea1ce96 100644 --- a/nifi-commons/nifi-security-utils/src/test/java/org/apache/nifi/security/XXEValidatorTest.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/java/org/apache/nifi/lookup/configuration2/XXEValidatorTest.java @@ -14,11 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.security; +package org.apache.nifi.lookup.configuration2; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.security.xml.XXEValidator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/nifi-commons/nifi-security-utils/src/test/resources/local_xxe_file.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/local_xxe_file.xml similarity index 100% rename from nifi-commons/nifi-security-utils/src/test/resources/local_xxe_file.xml rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/local_xxe_file.xml diff --git a/nifi-commons/nifi-security-utils/src/test/resources/multiline_xxe_file.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/multiline_xxe_file.xml similarity index 100% rename from nifi-commons/nifi-security-utils/src/test/resources/multiline_xxe_file.xml rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/multiline_xxe_file.xml diff --git a/nifi-commons/nifi-security-utils/src/test/resources/no_xxe.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/no_xxe.xml similarity index 100% rename from nifi-commons/nifi-security-utils/src/test/resources/no_xxe.xml rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/no_xxe.xml diff --git a/nifi-commons/nifi-security-utils/src/test/resources/remote_xxe_file.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/remote_xxe_file.xml similarity index 100% rename from nifi-commons/nifi-security-utils/src/test/resources/remote_xxe_file.xml rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/remote_xxe_file.xml diff --git a/nifi-commons/nifi-security-utils/src/test/resources/whitespace_xxe_file.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/whitespace_xxe_file.xml similarity index 100% rename from nifi-commons/nifi-security-utils/src/test/resources/whitespace_xxe_file.xml rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/whitespace_xxe_file.xml diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml index e851565994..8bbe871734 100755 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml @@ -51,6 +51,11 @@ nifi-syslog-utils 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + com.jayway.jsonpath json-path diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/windowsevent/WindowsEventLogRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/windowsevent/WindowsEventLogRecordReader.java index cf1e23e10a..c46299d11a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/windowsevent/WindowsEventLogRecordReader.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/windowsevent/WindowsEventLogRecordReader.java @@ -32,14 +32,17 @@ import org.apache.nifi.serialization.record.util.DataTypeUtils; import org.apache.nifi.stream.io.NonCloseableInputStream; import org.apache.nifi.util.StringUtils; import org.apache.nifi.xml.inference.XmlSchemaInference; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider; import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.Attribute; import javax.xml.stream.events.Characters; import javax.xml.stream.events.StartElement; import javax.xml.stream.events.XMLEvent; +import javax.xml.transform.stream.StreamSource; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; @@ -144,18 +147,13 @@ public class WindowsEventLogRecordReader implements RecordReader { LAZY_TIMESTAMP_FORMAT = () -> tsf; final FilterInputStream inputStream; - final XMLInputFactory xmlInputFactory; + final XMLEventReaderProvider provider = new StandardXMLEventReaderProvider(); try { - xmlInputFactory = XMLInputFactory.newInstance(); - // Avoid XXE Vulnerabilities - xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false); - inputStream = new NonCloseableInputStream(in); inputStream.mark(Integer.MAX_VALUE); - xmlEventReader = xmlInputFactory.createXMLEventReader(inputStream); + xmlEventReader = provider.getEventReader(new StreamSource(inputStream)); xmlSchemaInference = new XmlSchemaInference(new TimeValueInference(dateFormat, timeFormat, timestampFormat)); - } catch (XMLStreamException e) { + } catch (final ProcessingException e) { throw new MalformedRecordException("Error creating XML Event reader from FlowFile input stream", e); } @@ -169,7 +167,7 @@ public class WindowsEventLogRecordReader implements RecordReader { try { // Restart the XML event stream and advance to the first Event tag inputStream.reset(); - xmlEventReader = xmlInputFactory.createXMLEventReader(inputStream); + xmlEventReader = provider.getEventReader(new StreamSource(inputStream)); if (isArray) { skipToNextStartTag(); } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java index fd6d23b3e2..9de1a6253a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java @@ -31,14 +31,16 @@ import org.apache.nifi.serialization.record.type.MapDataType; import org.apache.nifi.serialization.record.type.RecordDataType; import org.apache.nifi.serialization.record.util.DataTypeUtils; import org.apache.nifi.util.StringUtils; +import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider; import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.Attribute; import javax.xml.stream.events.Characters; import javax.xml.stream.events.StartElement; import javax.xml.stream.events.XMLEvent; +import javax.xml.transform.stream.StreamSource; import java.io.IOException; import java.io.InputStream; import java.text.DateFormat; @@ -82,13 +84,8 @@ public class XMLRecordReader implements RecordReader { LAZY_TIMESTAMP_FORMAT = () -> tsf; try { - final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); - - // Avoid XXE Vulnerabilities - xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false); - - xmlEventReader = xmlInputFactory.createXMLEventReader(in); + final XMLEventReaderProvider provider = new StandardXMLEventReaderProvider(); + xmlEventReader = provider.getEventReader(new StreamSource(in)); if (isArray) { skipNextStartTag(); diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java index 8352aed08d..f77a61422a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java @@ -17,14 +17,17 @@ package org.apache.nifi.xml.inference; import org.apache.nifi.schema.inference.RecordSource; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider; import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.Attribute; import javax.xml.stream.events.Characters; import javax.xml.stream.events.StartElement; import javax.xml.stream.events.XMLEvent; +import javax.xml.transform.stream.StreamSource; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; @@ -40,18 +43,13 @@ public class XmlRecordSource implements RecordSource { public XmlRecordSource(final InputStream in, final String contentFieldName, final boolean ignoreWrapper) throws IOException { this.contentFieldName = contentFieldName; try { - final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); - - // Avoid XXE Vulnerabilities - xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false); - - xmlEventReader = xmlInputFactory.createXMLEventReader(in); + final XMLEventReaderProvider provider = new StandardXMLEventReaderProvider(); + xmlEventReader = provider.getEventReader(new StreamSource(in)); if (ignoreWrapper) { readStartElement(); } - } catch (XMLStreamException e) { + } catch (final ProcessingException|XMLStreamException e) { throw new IOException("Could not parse XML", e); } } diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/pom.xml b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/pom.xml index ef6867d2fd..cfa89daed1 100644 --- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/pom.xml +++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/pom.xml @@ -26,7 +26,7 @@ org.apache.nifi - nifi-security-utils + nifi-xml-processing 1.17.0-SNAPSHOT diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/src/main/java/org/apache/nifi/update/attributes/serde/CriteriaSerDe.java b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/src/main/java/org/apache/nifi/update/attributes/serde/CriteriaSerDe.java index 6ed0bed174..1d635e5fef 100644 --- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/src/main/java/org/apache/nifi/update/attributes/serde/CriteriaSerDe.java +++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-model/src/main/java/org/apache/nifi/update/attributes/serde/CriteriaSerDe.java @@ -26,12 +26,15 @@ import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import org.apache.nifi.security.xml.XmlUtils; +import javax.xml.transform.stream.StreamSource; + import org.apache.nifi.update.attributes.Criteria; import org.apache.nifi.update.attributes.FlowFilePolicy; import org.apache.nifi.update.attributes.Rule; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; /** * @@ -121,13 +124,14 @@ public class CriteriaSerDe { try { // deserialize the binding final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - XMLStreamReader xsr = XmlUtils.createSafeReader(new ByteArrayInputStream(string.getBytes(StandardCharsets.UTF_8))); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + XMLStreamReader xsr = provider.getStreamReader(new StreamSource(new ByteArrayInputStream(string.getBytes(StandardCharsets.UTF_8)))); final JAXBElement element = unmarshaller.unmarshal(xsr, CriteriaBinding.class); // create the criteria from the binding final CriteriaBinding binding = element.getValue(); criteria = new Criteria(binding.getFlowFilePolicy(), binding.getRules()); - } catch (final JAXBException | XMLStreamException e) { + } catch (final JAXBException | ProcessingException e) { throw new IllegalArgumentException(e); } } diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml b/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml index ea9d49356c..ddb2a298fd 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml @@ -211,6 +211,11 @@ nifi-h2-database-migrator 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + javax.servlet javax.servlet-api diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java index 3eddf9b028..8db74eb74b 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java @@ -27,7 +27,8 @@ import org.apache.nifi.registry.security.authentication.annotation.IdentityProvi import org.apache.nifi.registry.security.authentication.generated.IdentityProviders; import org.apache.nifi.registry.security.authentication.generated.Property; import org.apache.nifi.registry.security.authentication.generated.Provider; -import org.apache.nifi.registry.security.util.XmlUtils; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -147,7 +148,8 @@ public class IdentityProviderFactory implements IdentityProviderLookup, Disposab final Schema schema = schemaFactory.newSchema(IdentityProviders.class.getResource(LOGIN_IDENTITY_PROVIDERS_XSD)); // attempt to unmarshal - XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(loginIdentityProvidersConfigurationFile)); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + XMLStreamReader xsr = provider.getStreamReader(new StreamSource(loginIdentityProvidersConfigurationFile)); final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(schema); final JAXBElement element = unmarshaller.unmarshal(xsr, IdentityProviders.class); diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AbstractPolicyBasedAuthorizer.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AbstractPolicyBasedAuthorizer.java index 0509a3806f..16a6ec706a 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AbstractPolicyBasedAuthorizer.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AbstractPolicyBasedAuthorizer.java @@ -20,15 +20,14 @@ import org.apache.nifi.registry.security.authorization.exception.AuthorizationAc import org.apache.nifi.registry.security.authorization.exception.UninheritableAuthorizationsException; import org.apache.nifi.registry.security.exception.SecurityProviderCreationException; import org.apache.nifi.registry.security.exception.SecurityProviderDestructionException; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; @@ -47,7 +46,6 @@ import java.util.Set; */ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer { - static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance(); static final XMLOutputFactory XML_OUTPUT_FACTORY = XMLOutputFactory.newInstance(); static final String USER_ELEMENT = "user"; @@ -378,8 +376,8 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); - final Document document = docBuilder.parse(in); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); // parse all the users and add them to the current authorizer @@ -402,7 +400,7 @@ public abstract class AbstractPolicyBasedAuthorizer implements ManagedAuthorizer Node policyNode = policyNodes.item(i); accessPolicies.add(parsePolicy((Element) policyNode)); } - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java index 738f003573..90a3f0b5a8 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java @@ -35,8 +35,10 @@ import org.apache.nifi.registry.security.exception.SecurityProviderCreationExcep import org.apache.nifi.registry.security.exception.SecurityProviderDestructionException; import org.apache.nifi.registry.security.identity.IdentityMapper; import org.apache.nifi.registry.security.util.ClassLoaderUtils; -import org.apache.nifi.registry.security.util.XmlUtils; import org.apache.nifi.registry.service.RegistryService; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.DisposableBean; @@ -45,7 +47,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.lang.Nullable; import org.springframework.transaction.annotation.Transactional; -import org.xml.sax.SAXException; import javax.sql.DataSource; import javax.xml.XMLConstants; @@ -53,7 +54,7 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; -import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -300,9 +301,11 @@ public class AuthorizerFactory implements UserGroupProviderLookup, AccessPolicyP // attempt to unmarshal final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(schema); - final JAXBElement element = unmarshaller.unmarshal(XmlUtils.createSafeReader(new StreamSource(authorizersConfigurationFile)), Authorizers.class); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader reader = provider.getStreamReader(new StreamSource(authorizersConfigurationFile)); + final JAXBElement element = unmarshaller.unmarshal(reader, Authorizers.class); return element.getValue(); - } catch (XMLStreamException | SAXException | JAXBException e) { + } catch (final ProcessingException | JAXBException e) { throw new Exception("Unable to load the authorizer configuration file at: " + authorizersConfigurationFile.getAbsolutePath(), e); } } else { diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/StandardManagedAuthorizer.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/StandardManagedAuthorizer.java index 58bcf55464..db2f7d28dd 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/StandardManagedAuthorizer.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/StandardManagedAuthorizer.java @@ -22,15 +22,14 @@ import org.apache.nifi.registry.security.authorization.exception.UninheritableAu import org.apache.nifi.registry.security.exception.SecurityProviderCreationException; import org.apache.nifi.registry.security.exception.SecurityProviderDestructionException; import org.apache.nifi.registry.util.PropertyValue; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; @@ -42,7 +41,6 @@ import java.util.Set; public class StandardManagedAuthorizer implements ManagedAuthorizer { - private static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance(); private static final XMLOutputFactory XML_OUTPUT_FACTORY = XMLOutputFactory.newInstance(); private static final String USER_GROUP_PROVIDER_ELEMENT = "userGroupProvider"; @@ -212,8 +210,8 @@ public class StandardManagedAuthorizer implements ManagedAuthorizer { final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); - final Document document = docBuilder.parse(in); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); final NodeList accessPolicyProviderList = rootElement.getElementsByTagName(ACCESS_POLICY_PROVIDER_ELEMENT); @@ -229,7 +227,7 @@ public class StandardManagedAuthorizer implements ManagedAuthorizer { final Node accessPolicyProvider = accessPolicyProviderList.item(0); final Node userGroupProvider = userGroupProviderList.item(0); return new FingerprintHolder(accessPolicyProvider.getTextContent(), userGroupProvider.getTextContent()); - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } } diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileAccessPolicyProvider.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileAccessPolicyProvider.java index 31e77a1835..3de86089a8 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileAccessPolicyProvider.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileAccessPolicyProvider.java @@ -37,6 +37,9 @@ import org.apache.nifi.registry.security.exception.SecurityProviderCreationExcep import org.apache.nifi.registry.security.exception.SecurityProviderDestructionException; import org.apache.nifi.registry.security.identity.IdentityMapper; import org.apache.nifi.registry.util.PropertyValue; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -51,9 +54,6 @@ import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; @@ -95,7 +95,6 @@ public class FileAccessPolicyProvider extends AbstractConfigurableAccessPolicyPr } } - private static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance(); private static final XMLOutputFactory XML_OUTPUT_FACTORY = XMLOutputFactory.newInstance(); private static final String POLICY_ELEMENT = "policy"; @@ -336,8 +335,8 @@ public class FileAccessPolicyProvider extends AbstractConfigurableAccessPolicyPr final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); - final Document document = docBuilder.parse(in); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); // parse all the policies and add them to the current access policy provider @@ -346,7 +345,7 @@ public class FileAccessPolicyProvider extends AbstractConfigurableAccessPolicyPr Node policyNode = policyNodes.item(i); policies.add(parsePolicy((Element) policyNode)); } - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileUserGroupProvider.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileUserGroupProvider.java index ba7ca9c5fb..c1679cf28a 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileUserGroupProvider.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/file/FileUserGroupProvider.java @@ -34,6 +34,9 @@ import org.apache.nifi.registry.security.exception.SecurityProviderCreationExcep import org.apache.nifi.registry.security.exception.SecurityProviderDestructionException; import org.apache.nifi.registry.security.identity.IdentityMapper; import org.apache.nifi.registry.util.PropertyValue; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -48,9 +51,6 @@ import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; @@ -92,7 +92,6 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { } } - private static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance(); private static final XMLOutputFactory XML_OUTPUT_FACTORY = XMLOutputFactory.newInstance(); private static final String USER_ELEMENT = "user"; @@ -482,8 +481,8 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); - final Document document = docBuilder.parse(in); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); // parse all the users and add them to the current user group provider @@ -499,7 +498,7 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { Node groupNode = groupNodes.item(i); groups.add(parseGroup((Element) groupNode)); } - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/util/XmlUtils.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/util/XmlUtils.java deleted file mode 100644 index 2caa8faa46..0000000000 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/util/XmlUtils.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.registry.security.util; - -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; -import javax.xml.transform.stream.StreamSource; -import java.io.InputStream; - -public class XmlUtils { - - public static XMLStreamReader createSafeReader(InputStream inputStream) throws XMLStreamException { - if (inputStream == null) { - throw new IllegalArgumentException("The provided input stream cannot be null"); - } - return createSafeReader(new StreamSource(inputStream)); - } - - public static XMLStreamReader createSafeReader(StreamSource source) throws XMLStreamException { - if (source == null) { - throw new IllegalArgumentException("The provided source cannot be null"); - } - - XMLInputFactory xif = XMLInputFactory.newFactory(); - xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); - return xif.createXMLStreamReader(source); - } -} diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/service/alias/RegistryUrlAliasService.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/service/alias/RegistryUrlAliasService.java index b9e6e74711..650fb01366 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/service/alias/RegistryUrlAliasService.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/service/alias/RegistryUrlAliasService.java @@ -21,9 +21,11 @@ import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.registry.properties.NiFiRegistryProperties; import org.apache.nifi.registry.provider.ProviderFactoryException; import org.apache.nifi.registry.provider.StandardProviderFactory; -import org.apache.nifi.registry.security.util.XmlUtils; import org.apache.nifi.registry.url.aliaser.generated.Alias; import org.apache.nifi.registry.url.aliaser.generated.Aliases; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.stream.StandardXMLStreamReaderProvider; +import org.apache.nifi.xml.processing.stream.XMLStreamReaderProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.xml.sax.SAXException; @@ -33,7 +35,7 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; -import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -84,9 +86,11 @@ public class RegistryUrlAliasService { final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(schema); - final JAXBElement element = unmarshaller.unmarshal(XmlUtils.createSafeReader(new StreamSource(configurationFile)), Aliases.class); + final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); + final XMLStreamReader reader = provider.getStreamReader(new StreamSource(configurationFile)); + final JAXBElement element = unmarshaller.unmarshal(reader, Aliases.class); return element.getValue().getAlias(); - } catch (SAXException | JAXBException | XMLStreamException e) { + } catch (final SAXException | JAXBException | ProcessingException e) { throw new ProviderFactoryException("Unable to load the registry alias configuration file at: " + configurationFile.getAbsolutePath(), e); } } else { diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/service/extension/docs/XmlValidator.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/service/extension/docs/XmlValidator.java index 41cb6578f5..dfcaac1800 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/service/extension/docs/XmlValidator.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/service/extension/docs/XmlValidator.java @@ -16,23 +16,25 @@ */ package org.apache.nifi.registry.service.extension.docs; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.junit.Assert; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.IOException; -import java.io.StringReader; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; public class XmlValidator { + private static final String DOCTYPE = ""; + + private static final String EMPTY = ""; public static void assertXmlValid(String xml) { + final String html = xml.replace(DOCTYPE, EMPTY); try { - final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - dbf.setNamespaceAware(true); - dbf.newDocumentBuilder().parse(new InputSource(new StringReader(xml))); - } catch (SAXException | IOException | ParserConfigurationException e) { + final DocumentProvider provider = new StandardDocumentProvider(); + provider.parse(new ByteArrayInputStream(html.getBytes(StandardCharsets.UTF_8))); + } catch (final ProcessingException e) { Assert.fail(e.getMessage()); } } @@ -40,8 +42,4 @@ public class XmlValidator { public static void assertContains(String original, String subword) { Assert.assertTrue(original + " did not contain: " + subword, original.contains(subword)); } - - public static void assertNotContains(String original, String subword) { - Assert.assertFalse(original + " did contain: " + subword, original.contains(subword)); - } } diff --git a/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml b/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml index 896526abcb..38b219cc02 100644 --- a/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml +++ b/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml @@ -55,6 +55,13 @@ provided + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + + provided + org.apache.nifi.registry nifi-registry-ranger-jersey-bundle diff --git a/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/src/main/java/org/apache/nifi/registry/ranger/RangerAuthorizer.java b/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/src/main/java/org/apache/nifi/registry/ranger/RangerAuthorizer.java index 6fa6fe2e4f..134b792e70 100644 --- a/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/src/main/java/org/apache/nifi/registry/ranger/RangerAuthorizer.java +++ b/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/src/main/java/org/apache/nifi/registry/ranger/RangerAuthorizer.java @@ -39,6 +39,9 @@ import org.apache.nifi.registry.security.authorization.exception.AuthorizationAc import org.apache.nifi.registry.security.authorization.exception.UninheritableAuthorizationsException; import org.apache.nifi.registry.security.exception.SecurityProviderCreationException; import org.apache.nifi.registry.util.PropertyValue; +import org.apache.nifi.xml.processing.ProcessingException; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; @@ -52,11 +55,7 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; @@ -80,8 +79,6 @@ public class RangerAuthorizer implements ManagedAuthorizer, AuthorizationAuditor private static final Logger logger = LoggerFactory.getLogger(RangerAuthorizer.class); - private static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance(); - private static final String USER_GROUP_PROVIDER_ELEMENT = "userGroupProvider"; static final String USER_GROUP_PROVIDER = "User Group Provider"; @@ -326,8 +323,8 @@ public class RangerAuthorizer implements ManagedAuthorizer, AuthorizationAuditor final StringWriter out = new StringWriter(); try { // create the document - final DocumentBuilder documentBuilder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); - final Document document = documentBuilder.newDocument(); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.newDocument(); // create the root element final Element managedRangerAuthorizationsElement = document.createElement("managedRangerAuthorizations"); @@ -344,7 +341,7 @@ public class RangerAuthorizer implements ManagedAuthorizer, AuthorizationAuditor final Transformer transformer = TransformerFactory.newInstance().newTransformer(); transformer.transform(new DOMSource(document), new StreamResult(out)); - } catch (ParserConfigurationException | TransformerException e) { + } catch (final ProcessingException | TransformerException e) { throw new AuthorizationAccessException("Unable to generate fingerprint", e); } @@ -355,8 +352,8 @@ public class RangerAuthorizer implements ManagedAuthorizer, AuthorizationAuditor final byte[] fingerprintBytes = fingerprint.getBytes(StandardCharsets.UTF_8); try (final ByteArrayInputStream in = new ByteArrayInputStream(fingerprintBytes)) { - final DocumentBuilder docBuilder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); - final Document document = docBuilder.parse(in); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(in); final Element rootElement = document.getDocumentElement(); final NodeList userGroupProviderList = rootElement.getElementsByTagName(USER_GROUP_PROVIDER_ELEMENT); @@ -366,7 +363,7 @@ public class RangerAuthorizer implements ManagedAuthorizer, AuthorizationAuditor final Node userGroupProvider = userGroupProviderList.item(0); return userGroupProvider.getTextContent(); - } catch (SAXException | ParserConfigurationException | IOException e) { + } catch (final ProcessingException | IOException e) { throw new AuthorizationAccessException("Unable to parse fingerprint", e); } } diff --git a/nifi-system-tests/nifi-system-test-suite/pom.xml b/nifi-system-tests/nifi-system-test-suite/pom.xml index 16b448a675..d08e92e715 100644 --- a/nifi-system-tests/nifi-system-test-suite/pom.xml +++ b/nifi-system-tests/nifi-system-test-suite/pom.xml @@ -166,6 +166,16 @@ nifi-bootstrap 1.17.0-SNAPSHOT + + org.apache.nifi + nifi-security-utils + 1.17.0-SNAPSHOT + + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + org.apache.nifi nifi-jetty-bundle diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java index c53bd0b3d9..d5c9a8a15f 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java @@ -20,7 +20,6 @@ import org.apache.nifi.controller.serialization.FlowEncodingVersion; import org.apache.nifi.controller.serialization.FlowFromDOMFactory; import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.encrypt.PropertyEncryptorFactory; -import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.tests.system.InstanceConfiguration; import org.apache.nifi.tests.system.NiFiInstance; import org.apache.nifi.tests.system.NiFiInstanceFactory; @@ -45,20 +44,19 @@ import org.apache.nifi.web.api.entity.ControllerServicesEntity; import org.apache.nifi.web.api.entity.NodeEntity; import org.apache.nifi.web.api.entity.ParameterEntity; import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -142,8 +140,8 @@ public class JoinClusterWithDifferentFlow extends NiFiSystemIT { final File confDir = backupFile.getParentFile(); final String loadedFlow = readFlow(new File(confDir, "flow.xml.gz")); - final DocumentBuilder documentBuilder = XmlUtils.createSafeDocumentBuilder(false); - final Document document = documentBuilder.parse(new InputSource(new StringReader(loadedFlow))); + final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); + final Document document = documentProvider.parse(new ByteArrayInputStream(loadedFlow.getBytes(StandardCharsets.UTF_8))); final Element rootElement = (Element) document.getElementsByTagName("flowController").item(0); final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(rootElement); diff --git a/nifi-toolkit/nifi-toolkit-flowanalyzer/pom.xml b/nifi-toolkit/nifi-toolkit-flowanalyzer/pom.xml index 1506a1fd85..5bdd2e628c 100644 --- a/nifi-toolkit/nifi-toolkit-flowanalyzer/pom.xml +++ b/nifi-toolkit/nifi-toolkit-flowanalyzer/pom.xml @@ -24,6 +24,11 @@ 1.17.0-SNAPSHOT compile + + org.apache.nifi + nifi-xml-processing + 1.17.0-SNAPSHOT + diff --git a/nifi-toolkit/nifi-toolkit-flowanalyzer/src/main/java/org/apache/nifi/toolkit/flowanalyzer/FlowAnalyzerDriver.java b/nifi-toolkit/nifi-toolkit-flowanalyzer/src/main/java/org/apache/nifi/toolkit/flowanalyzer/FlowAnalyzerDriver.java index 3b9a8463ac..6b5e23802f 100644 --- a/nifi-toolkit/nifi-toolkit-flowanalyzer/src/main/java/org/apache/nifi/toolkit/flowanalyzer/FlowAnalyzerDriver.java +++ b/nifi-toolkit/nifi-toolkit-flowanalyzer/src/main/java/org/apache/nifi/toolkit/flowanalyzer/FlowAnalyzerDriver.java @@ -21,8 +21,8 @@ import java.io.InputStream; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.zip.GZIPInputStream; -import javax.xml.parsers.DocumentBuilder; -import org.apache.nifi.security.xml.XmlUtils; +import org.apache.nifi.xml.processing.parsers.DocumentProvider; +import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -71,8 +71,8 @@ public class FlowAnalyzerDriver { System.out.println("Using flow=" + input); try { - DocumentBuilder documentBuilder = XmlUtils.createSafeDocumentBuilder(false); - Document document = documentBuilder.parse(gzipStream); + final DocumentProvider documentProvider = new StandardDocumentProvider(); + Document document = documentProvider.parse(gzipStream); NodeList connectionNode = document.getElementsByTagName(CONST_XMLNODE_CONNECTION); for (int x = 0; x < connectionNode.getLength(); x++) {