Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-02-07 18:48:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-11126 Delete MultiPart files in ListenHTTP after processing

Browse Source 
Signed-off-by: Chris Sampson <chris.sampson82@gmail.com>

This closes #6915
This commit is contained in:
exceptionfactory 2023-02-01 12:39:07 -06:00 committed by Chris Sampson
parent d46aa89788
commit 8bc7593d34
No known key found for this signature in database
GPG Key ID: 546AEB0826587237
2 changed files with 31 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/servlets/ListenHTTPServlet.java
View File

@ -62,11 +62,9 @@ import java.io.OutputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.cert.X509Certificate; import java.security.cert.X509Certificate;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
@ -246,7 +244,11 @@ public class ListenHTTPServlet extends HttpServlet {
Set<FlowFile> flowFileSet; Set<FlowFile> flowFileSet;
if (StringUtils.isNotBlank(request.getContentType()) && request.getContentType().contains("multipart/form-data")) { if (StringUtils.isNotBlank(request.getContentType()) && request.getContentType().contains("multipart/form-data")) {
try {
flowFileSet = handleMultipartRequest(request, session, foundSubject, foundIssuer); flowFileSet = handleMultipartRequest(request, session, foundSubject, foundIssuer);
} finally {
deleteMultiPartFiles(request);
}
} else { } else {
flowFileSet = handleRequest(request, session, foundSubject, foundIssuer, destinationIsLegacyNiFi, contentType, in); flowFileSet = handleRequest(request, session, foundSubject, foundIssuer, destinationIsLegacyNiFi, contentType, in);
} }
@ -256,6 +258,16 @@ public class ListenHTTPServlet extends HttpServlet {
} }
} }
private void deleteMultiPartFiles(final HttpServletRequest request) {
try {
for (final Part part : request.getParts()) {
part.delete();
}
} catch (final Exception e) {
logger.warn("Delete MultiPart temporary files failed", e);
}
}
private void handleException(final HttpServletRequest request, final HttpServletResponse response, private void handleException(final HttpServletRequest request, final HttpServletResponse response,
final ProcessSession session, final String foundSubject, final String foundIssuer, final Throwable t) throws IOException { final ProcessSession session, final String foundSubject, final String foundIssuer, final Throwable t) throws IOException {
session.rollback(); session.rollback();
@ -270,20 +282,18 @@ public class ListenHTTPServlet extends HttpServlet {
private Set<FlowFile> handleMultipartRequest(HttpServletRequest request, ProcessSession session, String foundSubject, String foundIssuer) private Set<FlowFile> handleMultipartRequest(HttpServletRequest request, ProcessSession session, String foundSubject, String foundIssuer)
throws IOException, IllegalStateException, ServletException { throws IOException, IllegalStateException, ServletException {
if (isRecordProcessing()) {
logger.debug("Record processing will not be utilized while processing multipart request. Request URI: {}", request.getRequestURI());
}
Set<FlowFile> flowFileSet = new HashSet<>(); Set<FlowFile> flowFileSet = new HashSet<>();
String tempDir = System.getProperty("java.io.tmpdir"); String tempDir = System.getProperty("java.io.tmpdir");
request.setAttribute(Request.MULTIPART_CONFIG_ELEMENT, new MultipartConfigElement(tempDir, multipartRequestMaxSize, multipartRequestMaxSize, multipartReadBufferSize)); request.setAttribute(Request.MULTIPART_CONFIG_ELEMENT, new MultipartConfigElement(tempDir, multipartRequestMaxSize, multipartRequestMaxSize, multipartReadBufferSize));
Collection<Part> requestParts = Collections.unmodifiableCollection(request.getParts());
final Iterator<Part> parts = requestParts.iterator();
int i = 0; int i = 0;
while (parts.hasNext()) { final Collection<Part> requestParts = request.getParts();
Part part = parts.next(); for (final Part part : requestParts) {
FlowFile flowFile = session.create(); FlowFile flowFile = session.create();
try (OutputStream flowFileOutputStream = session.write(flowFile)) { try (
StreamUtils.copy(part.getInputStream(), flowFileOutputStream); OutputStream flowFileOutputStream = session.write(flowFile);
InputStream partInputStream = part.getInputStream()
) {
StreamUtils.copy(partInputStream, flowFileOutputStream);
} }
flowFile = saveRequestDetailsAsAttributes(request, session, foundSubject, foundIssuer, flowFile); flowFile = saveRequestDetailsAsAttributes(request, session, foundSubject, foundIssuer, flowFile);
flowFile = savePartDetailsAsAttributes(session, part, flowFile, i, requestParts.size()); flowFile = savePartDetailsAsAttributes(session, part, flowFile, i, requestParts.size());
@ -333,9 +343,6 @@ public class ListenHTTPServlet extends HttpServlet {
hasMoreData.set(false); hasMoreData.set(false);
} }
} else { } else {
if (isRecordProcessing()) {
logger.debug("Record processing will not be utilized while processing with unpackager. Request URI: {}", request.getRequestURI());
}
attributes.putAll(unpackager.unpackageFlowFile(in, bos)); attributes.putAll(unpackager.unpackageFlowFile(in, bos));
if (destinationIsLegacyNiFi) { if (destinationIsLegacyNiFi) {

9
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestListenHTTP.java
View File

@ -23,6 +23,7 @@ import java.net.InetSocketAddress;
import java.net.Socket; import java.net.Socket;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.security.GeneralSecurityException; import java.security.GeneralSecurityException;
import java.time.Duration; import java.time.Duration;
@ -696,6 +697,7 @@ public class TestListenHTTP {
runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort)); runner.setProperty(ListenHTTP.PORT, Integer.toString(availablePort));
runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH); runner.setProperty(ListenHTTP.BASE_PATH, HTTP_BASE_PATH);
runner.setProperty(ListenHTTP.RETURN_CODE, Integer.toString(HttpServletResponse.SC_OK)); runner.setProperty(ListenHTTP.RETURN_CODE, Integer.toString(HttpServletResponse.SC_OK));
runner.setProperty(ListenHTTP.MULTIPART_READ_BUFFER_SIZE, "10 bytes");
final SSLContextService sslContextService = runner.getControllerService(SSL_CONTEXT_SERVICE_IDENTIFIER, SSLContextService.class); final SSLContextService sslContextService = runner.getControllerService(SSL_CONTEXT_SERVICE_IDENTIFIER, SSLContextService.class);
final boolean isSecure = (sslContextService != null); final boolean isSecure = (sslContextService != null);
@ -769,6 +771,13 @@ public class TestListenHTTP {
mff.assertAttributeExists("http.multipart.fragments.sequence.number"); mff.assertAttributeExists("http.multipart.fragments.sequence.number");
mff.assertAttributeEquals("http.multipart.fragments.total.number", "5"); mff.assertAttributeEquals("http.multipart.fragments.total.number", "5");
mff.assertAttributeExists("http.headers.multipart.content-disposition"); mff.assertAttributeExists("http.headers.multipart.content-disposition");
final Path tempDirectoryPath = Paths.get(System.getProperty("java.io.tmpdir"));
final long multiPartTempFiles = Files.find(tempDirectoryPath, 1,
(filePath, fileAttributes) -> filePath.getFileName().toString().startsWith("MultiPart")
).count();
final String multiPartMessage = String.format("MultiPart files found in temporary directory [%s]", tempDirectoryPath);
assertEquals(0, multiPartTempFiles, multiPartMessage);
} }
private byte[] generateRandomBinaryData() { private byte[] generateRandomBinaryData() {