From 0458b6432dd010e6727ac5df7d96310e22c5f0a4 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Tue, 28 Jun 2022 08:53:38 -0500 Subject: [PATCH] NIFI-10158 Corrected ListFTP expression support for Hostname and Port Signed-off-by: Nathan Gough This closes #6162. --- .../ftp/StandardFTPClientProvider.java | 13 ++++- .../nifi/processors/standard/TestFTP.java | 55 ++++++++++++++----- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ftp/StandardFTPClientProvider.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ftp/StandardFTPClientProvider.java index debc5b3075..726d1e34ea 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ftp/StandardFTPClientProvider.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ftp/StandardFTPClientProvider.java @@ -79,11 +79,16 @@ public class StandardFTPClientProvider implements FTPClientProvider { final boolean attributesEmpty = attributes.isEmpty(); + // Evaluate Hostname and Port properties based on the presence of attributes because ListFTP does not support FlowFile attributes final PropertyValue hostnameProperty = context.getProperty(HOSTNAME); - final String hostname = attributesEmpty ? hostnameProperty.getValue() : hostnameProperty.evaluateAttributeExpressions(attributes).getValue(); + final String hostname = attributesEmpty + ? hostnameProperty.evaluateAttributeExpressions().getValue() + : hostnameProperty.evaluateAttributeExpressions(attributes).getValue(); final PropertyValue portProperty = context.getProperty(PORT); - final int port = attributesEmpty ? portProperty.asInteger() : portProperty.evaluateAttributeExpressions(attributes).asInteger(); + final int port = attributesEmpty + ? portProperty.evaluateAttributeExpressions().asInteger() + : portProperty.evaluateAttributeExpressions(attributes).asInteger(); final String address = String.format(ADDRESS_FORMAT, hostname, port); try { @@ -96,7 +101,9 @@ public class StandardFTPClientProvider implements FTPClientProvider { } final PropertyValue usernameProperty = context.getProperty(USERNAME); - final String username = attributesEmpty ? usernameProperty.getValue() : usernameProperty.evaluateAttributeExpressions(attributes).getValue(); + final String username = attributesEmpty + ? usernameProperty.evaluateAttributeExpressions().getValue() + : usernameProperty.evaluateAttributeExpressions(attributes).getValue(); final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions(attributes).getValue(); try { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java index 418b1528d6..ee8cfe9a40 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java @@ -50,6 +50,8 @@ import org.mockftpserver.fake.filesystem.WindowsFakeFileSystem; public class TestFTP { + private static final String LOCALHOST_ADDRESS = "127.0.0.1"; + final FakeFtpServer fakeFtpServer = new FakeFtpServer(); final String username = "nifi-ftp-user"; final String password = "Test test test chocolate"; @@ -126,9 +128,9 @@ public class TestFTP { } @Test - public void basicFileUpload() throws IOException { + public void testPutFtp() throws IOException { TestRunner runner = TestRunners.newTestRunner(PutFTP.class); - runner.setProperty(FTPTransfer.HOSTNAME, "localhost"); + runner.setProperty(FTPTransfer.HOSTNAME, LOCALHOST_ADDRESS); runner.setProperty(FTPTransfer.USERNAME, username); runner.setProperty(FTPTransfer.PASSWORD, password); runner.setProperty(FTPTransfer.PORT, Integer.toString(ftpPort)); @@ -147,7 +149,7 @@ public class TestFTP { } @Test - public void basicProvenanceEventTest() throws IOException { + public void testPutFtpProvenanceEvents() throws IOException { TestRunner runner = TestRunners.newTestRunner(PutFTP.class); runner.setProperty(FTPTransfer.HOSTNAME, "localhost"); @@ -166,7 +168,7 @@ public class TestFTP { try (FileInputStream fis = new FileInputStream("src/test/resources/hello.txt")) { Map attributes = new HashMap<>(); attributes.put(CoreAttributes.FILENAME.key(), "hello.txt"); - attributes.put("transfer-host", "127.0.0.1"); + attributes.put("transfer-host", LOCALHOST_ADDRESS); runner.enqueue(fis, attributes); runner.run(); } @@ -198,7 +200,7 @@ public class TestFTP { } @Test - public void basicFileGet() { + public void testGetFtp() { FileSystem results = fakeFtpServer.getFileSystem(); FileEntry sampleFile = new FileEntry("c:\\data\\randombytes-2"); @@ -209,7 +211,7 @@ public class TestFTP { Assertions.assertTrue(results.exists("c:\\data\\randombytes-2")); TestRunner runner = TestRunners.newTestRunner(GetFTP.class); - runner.setProperty(FTPTransfer.HOSTNAME, "localhost"); + runner.setProperty(FTPTransfer.HOSTNAME, LOCALHOST_ADDRESS); runner.setProperty(FTPTransfer.USERNAME, username); runner.setProperty(FTPTransfer.PASSWORD, password); runner.setProperty(FTPTransfer.PORT, Integer.toString(ftpPort)); @@ -222,7 +224,7 @@ public class TestFTP { } @Test - public void basicFileFetch() { + public void testFetchFtp() { FileSystem results = fakeFtpServer.getFileSystem(); FileEntry sampleFile = new FileEntry("c:\\data\\randombytes-2"); @@ -255,9 +257,9 @@ public class TestFTP { } @Test - public void testFetchFileNotFound() { + public void testFetchFtpFileNotFound() { final TestRunner runner = TestRunners.newTestRunner(FetchFTP.class); - runner.setProperty(FetchFTP.HOSTNAME, "127.0.0.1"); + runner.setProperty(FetchFTP.HOSTNAME, LOCALHOST_ADDRESS); runner.setProperty(FetchFTP.USERNAME, username); runner.setProperty(FTPTransfer.PASSWORD, password); runner.setProperty(FTPTransfer.PORT, Integer.toString(ftpPort)); @@ -270,7 +272,7 @@ public class TestFTP { } @Test - public void testFetchFilePermissionDenied() { + public void testFetchFtpFilePermissionDenied() { final FileSystem fs = fakeFtpServer.getFileSystem(); final FileEntry restrictedFileEntry = new FileEntry("c:\\data\\restricted"); @@ -278,7 +280,7 @@ public class TestFTP { fs.add(restrictedFileEntry); final TestRunner runner = TestRunners.newTestRunner(FetchFTP.class); - runner.setProperty(FetchFTP.HOSTNAME, "127.0.0.1"); + runner.setProperty(FetchFTP.HOSTNAME, LOCALHOST_ADDRESS); runner.setProperty(FetchFTP.USERNAME, username); runner.setProperty(FTPTransfer.PASSWORD, password); runner.setProperty(FTPTransfer.PORT, Integer.toString(ftpPort)); @@ -290,10 +292,33 @@ public class TestFTP { runner.assertAllFlowFilesTransferred(FetchFTP.REL_PERMISSION_DENIED); } + @Test + public void testListFtpHostPortVariablesFileFound() { + final FileSystem fs = fakeFtpServer.getFileSystem(); + + final FileEntry fileEntry = new FileEntry("c:\\data\\found"); + fs.add(fileEntry); + + final TestRunner runner = TestRunners.newTestRunner(ListFTP.class); + runner.setVariable("host", LOCALHOST_ADDRESS); + runner.setVariable("port", Integer.toString(ftpPort)); + + runner.setProperty(ListFTP.HOSTNAME, "${host}"); + runner.setProperty(FTPTransfer.PORT, "${port}"); + runner.setProperty(ListFTP.USERNAME, username); + runner.setProperty(FTPTransfer.PASSWORD, password); + runner.setProperty(ListFile.TARGET_SYSTEM_TIMESTAMP_PRECISION, ListFile.PRECISION_MILLIS); + + runner.enqueue(new byte[0]); + runner.run(); + + runner.assertTransferCount(ListFTP.REL_SUCCESS, 1); + } + @Test @EnabledIfSystemProperty(named = "file.encoding", matches = "UTF-8", disabledReason = "org.mockftpserver does not support specification of charset") - public void basicFileFetchWithUTF8FileName() { + public void testFetchFtpUnicodeFileName() { FileSystem fs = fakeFtpServer.getFileSystem(); FileEntry sampleFile = new FileEntry("c:\\data\\őűőű.txt"); @@ -301,7 +326,7 @@ public class TestFTP { fs.add(sampleFile); TestRunner runner = TestRunners.newTestRunner(FetchFTP.class); - runner.setProperty(FetchFTP.HOSTNAME, "localhost"); + runner.setProperty(FetchFTP.HOSTNAME, LOCALHOST_ADDRESS); runner.setProperty(FetchFTP.USERNAME, username); runner.setProperty(FTPTransfer.PASSWORD, password); runner.setProperty(FTPTransfer.PORT, String.valueOf(ftpPort)); @@ -320,7 +345,7 @@ public class TestFTP { } @Test - public void basicFileList() throws InterruptedException { + public void testListFtp() throws InterruptedException { FileSystem results = fakeFtpServer.getFileSystem(); FileEntry sampleFile = new FileEntry("c:\\data\\randombytes-2"); @@ -331,7 +356,7 @@ public class TestFTP { Assertions.assertTrue(results.exists("c:\\data\\randombytes-2")); TestRunner runner = TestRunners.newTestRunner(ListFTP.class); - runner.setProperty(ListFTP.HOSTNAME, "localhost"); + runner.setProperty(ListFTP.HOSTNAME, LOCALHOST_ADDRESS); runner.setProperty(ListFTP.USERNAME, username); runner.setProperty(FTPTransfer.PASSWORD, password); runner.setProperty(FTPTransfer.PORT, Integer.toString(ftpPort));