From 28067a29fd13cdf8e21b440fc65c6dd67872522f Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Mon, 12 Feb 2018 21:10:16 -0800 Subject: [PATCH] NIFI-4869 Added SAX utility method for SplitXML processor. Added unit tests. Added test resources to RAT exclude list. This closes #2466 --- .../apache/nifi/security/xml/XmlUtils.java | 27 +- .../nifi-standard-processors/pom.xml | 2 + .../nifi/processors/standard/SplitXml.java | 24 +- .../processors/standard/SplitXmlTest.groovy | 85 +++++++ .../src/test/resources/xxe_from_report.xml | 2 + .../src/test/resources/xxe_template.xml | 230 ++++++++++++++++++ 6 files changed, 354 insertions(+), 16 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/SplitXmlTest.groovy create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_from_report.xml create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_template.xml diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java index 99c90a6d53..e384c5b30a 100644 --- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java @@ -16,11 +16,17 @@ */ package org.apache.nifi.security.xml; +import java.io.InputStream; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.transform.stream.StreamSource; -import java.io.InputStream; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; public class XmlUtils { @@ -41,4 +47,23 @@ public class XmlUtils { xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); return xif.createXMLStreamReader(source); } + + public static XMLReader createSafeSaxReader(SAXParserFactory saxParserFactory, ContentHandler contentHandler) throws SAXException, ParserConfigurationException { + if (saxParserFactory == null) { + throw new IllegalArgumentException("The provided SAX parser factory cannot be null"); + } + + if (contentHandler == null) { + throw new IllegalArgumentException("The provided SAX content handler cannot be null"); + } + + saxParserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); + saxParserFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + + SAXParser saxParser = saxParserFactory.newSAXParser(); + XMLReader xmlReader = saxParser.getXMLReader(); + xmlReader.setContentHandler(contentHandler); + + return xmlReader; + } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 3cdd7879ea..0fffbb8522 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -515,6 +515,8 @@ src/main/java/org/apache/nifi/security/util/crypto/bcrypt/BCrypt.java + src/test/resources/xxe_template.xml + src/test/resources/xxe_from_report.xml diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java index 502f7f3506..de513c8ba4 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitXml.java @@ -16,6 +16,12 @@ */ package org.apache.nifi.processors.standard; +import static org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_COUNT; +import static org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_ID; +import static org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_INDEX; +import static org.apache.nifi.flowfile.attributes.FragmentAttributes.SEGMENT_ORIGINAL_FILENAME; +import static org.apache.nifi.flowfile.attributes.FragmentAttributes.copyAttributesToOriginal; + import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; @@ -28,11 +34,8 @@ import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - import javax.xml.parsers.ParserConfigurationException; -import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; - import org.apache.commons.lang3.StringEscapeUtils; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.InputRequirement; @@ -54,7 +57,7 @@ import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.processors.standard.util.XmlElementNotifier; -import org.apache.nifi.stream.io.BufferedInputStream; +import org.apache.nifi.security.xml.XmlUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.Attributes; @@ -64,12 +67,6 @@ import org.xml.sax.Locator; import org.xml.sax.SAXException; import org.xml.sax.XMLReader; -import static org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_COUNT; -import static org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_ID; -import static org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_INDEX; -import static org.apache.nifi.flowfile.attributes.FragmentAttributes.SEGMENT_ORIGINAL_FILENAME; -import static org.apache.nifi.flowfile.attributes.FragmentAttributes.copyAttributesToOriginal; - @EventDriven @SideEffectFree @SupportsBatching @@ -175,12 +172,9 @@ public class SplitXml extends AbstractProcessor { final AtomicBoolean failed = new AtomicBoolean(false); session.read(original, rawIn -> { - try (final InputStream in = new BufferedInputStream(rawIn)) { - SAXParser saxParser = null; + try (final InputStream in = new java.io.BufferedInputStream(rawIn)) { try { - saxParser = saxParserFactory.newSAXParser(); - final XMLReader reader = saxParser.getXMLReader(); - reader.setContentHandler(parser); + final XMLReader reader = XmlUtils.createSafeSaxReader(saxParserFactory, parser); reader.parse(new InputSource(in)); } catch (final ParserConfigurationException | SAXException e) { logger.error("Unable to parse {} due to {}", new Object[]{original, e}); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/SplitXmlTest.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/SplitXmlTest.groovy new file mode 100644 index 0000000000..f04dca685e --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/SplitXmlTest.groovy @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.standard + +import org.apache.nifi.util.TestRunner +import org.apache.nifi.util.TestRunners +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.nio.file.Paths + + +@RunWith(JUnit4.class) +class SplitXmlTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(SplitXmlTest.class) + + @BeforeClass + static void setUpOnce() throws Exception { + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + + } + + @After + void tearDown() throws Exception { + + } + + @Test + void testShouldHandleXXEInTemplate() { + // Arrange + final String XXE_TEMPLATE_FILEPATH = "src/test/resources/xxe_template.xml" + final TestRunner runner = TestRunners.newTestRunner(new SplitXml()) + runner.setProperty(SplitXml.SPLIT_DEPTH, "3") + runner.enqueue(Paths.get(XXE_TEMPLATE_FILEPATH)) + + // Act + runner.run() + logger.info("SplitXML processor ran") + + // Assert + runner.assertAllFlowFilesTransferred(SplitXml.REL_FAILURE) + } + + @Test + void testShouldHandleRemoteCallXXE() { + // Arrange + final String XXE_TEMPLATE_FILEPATH = "src/test/resources/xxe_from_report.xml" + final TestRunner runner = TestRunners.newTestRunner(new SplitXml()) + runner.setProperty(SplitXml.SPLIT_DEPTH, "3") + runner.enqueue(Paths.get(XXE_TEMPLATE_FILEPATH)) + + // Act + runner.run() + logger.info("SplitXML processor ran") + + // Assert + runner.assertAllFlowFilesTransferred(SplitXml.REL_FAILURE) + } +} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_from_report.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_from_report.xml new file mode 100644 index 0000000000..42b22a0f3f --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_from_report.xml @@ -0,0 +1,2 @@ + +1 \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_template.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_template.xml new file mode 100644 index 0000000000..82674e0e1a --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/xxe_template.xml @@ -0,0 +1,230 @@ +]> +