Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-6699 Corrected SFTP symbolic link handling (#5744) 

- Added SFTP stat() request on symbolic links to check directory status
- Refactored and renamed TestServerSFTPTransfer class
This commit is contained in:
exceptionfactory 2022-02-04 17:43:42 -06:00 committed by GitHub
parent bab4309905
commit ce66cf41e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 227 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
View File

@ -41,7 +41,6 @@ import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.OutputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.standard.ssh.PatchedSFTPEngine;
import org.apache.nifi.processors.standard.ssh.SSHClientProvider;
@ -54,7 +53,6 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.DateFormat;
@ -74,6 +72,10 @@ import java.util.stream.Collectors;
public class SFTPTransfer implements FileTransfer {
private static final SSHClientProvider SSH_CLIENT_PROVIDER = new StandardSSHClientProvider();
private static final String DOT_PREFIX = ".";
private static final String RELATIVE_CURRENT_DIRECTORY = DOT_PREFIX;
private static final String RELATIVE_PARENT_DIRECTORY = "..";
private static final Set<String> DEFAULT_KEY_ALGORITHM_NAMES;
private static final Set<String> DEFAULT_CIPHER_NAMES;
private static final Set<String> DEFAULT_MESSAGE_AUTHENTICATION_CODE_NAMES;
@ -278,7 +280,7 @@ public class SFTPTransfer implements FileTransfer {
final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean();
final boolean symlink = ctx.getProperty(FileTransfer.FOLLOW_SYMLINK).asBoolean();
final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue();
final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex);
final Pattern fileFilterPattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex);
final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue();
final Pattern pathPattern = (!recurse || pathFilterRegex == null) ? null : Pattern.compile(pathFilterRegex);
final String remotePath = ctx.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions().getValue();
@ -286,7 +288,7 @@ public class SFTPTransfer implements FileTransfer {
// check if this directory path matches the PATH_FILTER_REGEX
boolean pathFilterMatches = true;
if (pathPattern != null) {
Path reldir = path == null ? Paths.get(".") : Paths.get(path);
Path reldir = path == null ? Paths.get(RELATIVE_CURRENT_DIRECTORY) : Paths.get(path);
if (remotePath != null) {
reldir = Paths.get(remotePath).relativize(reldir);
}
@ -298,26 +300,25 @@ public class SFTPTransfer implements FileTransfer {
}
final SFTPClient sftpClient = getSFTPClient(null);
final boolean isPathMatch = pathFilterMatches;
final boolean pathMatched = pathFilterMatches;
final boolean filteringDisabled = !applyFilters;
//subDirs list is used for both 'sub directories' and 'symlinks'
final List<RemoteResourceInfo> subDirs = new ArrayList<>();
try {
final RemoteResourceFilter filter = (entry) -> {
final String entryFilename = entry.getName();
// skip over 'this directory' and 'parent directory' special files regardless of ignoring dot files
if (entryFilename.equals(".") || entryFilename.equals("..")) {
if (RELATIVE_CURRENT_DIRECTORY.equals(entryFilename) || RELATIVE_PARENT_DIRECTORY.equals(entryFilename)) {
return false;
}
// skip files and directories that begin with a dot if we're ignoring them
if (ignoreDottedFiles && entryFilename.startsWith(".")) {
if (ignoreDottedFiles && entryFilename.startsWith(DOT_PREFIX)) {
return false;
}
// if is a directory and we're supposed to recurse OR if is a link and we're supposed to follow symlink
if ((recurse && entry.isDirectory()) || (symlink && (entry.getAttributes().getType() == FileMode.Type.SYMLINK))){
if (isIncludedDirectory(entry, recurse, symlink)) {
subDirs.add(entry);
return false;
}
@ -328,9 +329,8 @@ public class SFTPTransfer implements FileTransfer {
return false;
}
// if is not a directory and is not a link and it matches FILE_FILTER_REGEX - then let's add it
if (!entry.isDirectory() && !(entry.getAttributes().getType() == FileMode.Type.SYMLINK) && (!applyFilters || isPathMatch)) {
if (pattern == null || !applyFilters || pattern.matcher(entryFilename).matches()) {
if (isIncludedFile(entry, symlink) && (filteringDisabled || pathMatched)) {
if (filteringDisabled || fileFilterPattern == null || fileFilterPattern.matcher(entryFilename).matches()) {
listing.add(newFileInfo(entry, path));
}
}
@ -339,7 +339,7 @@ public class SFTPTransfer implements FileTransfer {
};
if (path == null || path.trim().isEmpty()) {
sftpClient.ls(".", filter);
sftpClient.ls(RELATIVE_CURRENT_DIRECTORY, filter);
} else {
sftpClient.ls(path, filter);
}
@ -370,6 +370,47 @@ public class SFTPTransfer implements FileTransfer {
}
/**
* Include remote resources when regular file found or when symbolic links are enabled and the resource is a link
*
* @param remoteResourceInfo Remote Resource Information
* @param symlinksEnabled Follow symbolic links enabled
* @return Included file status
*/
private boolean isIncludedFile(final RemoteResourceInfo remoteResourceInfo, final boolean symlinksEnabled) {
return remoteResourceInfo.isRegularFile() || (symlinksEnabled && isSymlink(remoteResourceInfo));
}
/**
* Include remote resources when recursion is enabled or when symbolic links are enabled and the resource is a directory link
*
* @param remoteResourceInfo Remote Resource Information
* @param recursionEnabled Recursion enabled status
* @param symlinksEnabled Follow symbolic links enabled
* @return Included directory status
*/
private boolean isIncludedDirectory(final RemoteResourceInfo remoteResourceInfo, final boolean recursionEnabled, final boolean symlinksEnabled) {
boolean includedDirectory = false;
if (remoteResourceInfo.isDirectory()) {
includedDirectory = recursionEnabled;
} else if (symlinksEnabled && isSymlink(remoteResourceInfo)) {
final String path = remoteResourceInfo.getPath();
try {
final FileAttributes pathAttributes = sftpClient.stat(path);
includedDirectory = FileMode.Type.DIRECTORY == pathAttributes.getMode().getType();
} catch (final IOException e) {
logger.warn("Read symbolic link attributes failed [{}]", path, e);
}
}
return includedDirectory;
}
private boolean isSymlink(final RemoteResourceInfo remoteResourceInfo) {
return FileMode.Type.SYMLINK == remoteResourceInfo.getAttributes().getType();
}
private FileInfo newFileInfo(final RemoteResourceInfo entry, String path) {
if (entry == null) {
return null;
@ -422,12 +463,7 @@ public class SFTPTransfer implements FileTransfer {
rf = sftpClient.open(remoteFileName);
rfis = rf.new ReadAheadRemoteFileInputStream(16);
final InputStream in = rfis;
resultFlowFile = session.write(origFlowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream out) throws IOException {
StreamUtils.copy(in, out);
}
});
resultFlowFile = session.write(origFlowFile, out -> StreamUtils.copy(in, out));
return resultFlowFile;
} catch (final SFTPException e) {
switch (e.getStatusCode()) {
@ -660,7 +696,7 @@ public class SFTPTransfer implements FileTransfer {
String tempFilename = ctx.getProperty(TEMP_FILENAME).evaluateAttributeExpressions(flowFile).getValue();
if (tempFilename == null) {
final boolean dotRename = ctx.getProperty(DOT_RENAME).asBoolean();
tempFilename = dotRename ? "." + filename : filename;
tempFilename = dotRename ? DOT_PREFIX + filename : filename;
}
final String tempPath = (path == null) ? tempFilename : (path.endsWith("/")) ? path + tempFilename : path + "/" + tempFilename;

374
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/ITestSFTPTransferWithSSHTestServer.java → nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestServerSFTPTransfer.java
View File

@ -16,30 +16,27 @@
*/
package org.apache.nifi.processors.standard.util;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.util.MockPropertyContext;
import org.apache.nifi.util.file.FileUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
import org.apache.sshd.server.SshServer;
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
import org.apache.sshd.sftp.server.SftpSubsystemFactory;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@ -47,27 +44,33 @@ import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class ITestSFTPTransferWithSSHTestServer {
public class TestServerSFTPTransfer {
private static final Logger LOGGER = LoggerFactory.getLogger(ITestSFTPTransferWithSSHTestServer.class);
private static final String LOCALHOST = "127.0.0.1";
private static final String SFTP_ROOT_DIR = "target/test-sftp-transfer-vfs";
private static final String USERNAME = "user";
private static final String PASSWORD = UUID.randomUUID().toString();
private static final String DIR_1 = "dir1";
private static final String DIR_2 = "dir2";
private static final String DIR_3 = "dir3";
private static final String DIR_4 = "dir4";
private static final String LINKED_DIRECTORY = "linked-directory";
private static final String LINKED_FILE = "linked-file";
private static final String EMPTY_DIRECTORY = "dir4";
private static final String DIR_1_CHILD_1 = "child1";
private static final String DIR_1_CHILD_2 = "child2";
@ -76,66 +79,45 @@ public class ITestSFTPTransferWithSSHTestServer {
private static final String FILE_2 = "file2.txt";
private static final String DOT_FILE = ".foo.txt";
private static SSHTestServer sshTestServer;
private static final boolean FILTERING_ENABLED = true;
@BeforeClass
public static void setupClass() throws IOException {
sshTestServer = new SSHTestServer();
sshTestServer.setVirtualFileSystemPath(SFTP_ROOT_DIR);
sshTestServer.startServer();
}
@TempDir
File serverDirectory;
@AfterClass
public static void cleanupClass() throws IOException {
sshTestServer.stopServer();
}
private SshServer sshServer;
@Before
@BeforeEach
public void setupFiles() throws IOException {
final File sftpRootDir = new File(SFTP_ROOT_DIR);
FileUtils.deleteFilesInDir(sftpRootDir, null, LOGGER, true, true);
writeFile(DIR_1, DIR_1_CHILD_1, FILE_1);
writeFile(DIR_1, DIR_1_CHILD_1, FILE_2);
writeFile(DIR_1, DIR_1_CHILD_1, DOT_FILE);
// create and initialize dir1/child1
initializeFile(SFTP_ROOT_DIR + "/" + DIR_1 + "/" + DIR_1_CHILD_1, FILE_1, "dir1 child1 file1");
initializeFile(SFTP_ROOT_DIR + "/" + DIR_1 + "/" + DIR_1_CHILD_1, FILE_2, "dir1 child1 file2");
initializeFile(SFTP_ROOT_DIR + "/" + DIR_1 + "/" + DIR_1_CHILD_1, DOT_FILE, "dir1 child1 foo");
writeFile(DIR_1, DIR_1_CHILD_2, FILE_1);
writeFile(DIR_1, DIR_1_CHILD_2, FILE_2);
writeFile(DIR_1, DIR_1_CHILD_2, DOT_FILE);
// create and initialize dir1/child2
initializeFile(SFTP_ROOT_DIR + "/" + DIR_1 + "/" + DIR_1_CHILD_2, FILE_1, "dir1 child2 file1");
initializeFile(SFTP_ROOT_DIR + "/" + DIR_1 + "/" + DIR_1_CHILD_2, FILE_2, "dir1 child2 file2");
initializeFile(SFTP_ROOT_DIR + "/" + DIR_1 + "/" + DIR_1_CHILD_2, DOT_FILE, "dir1 child2 foo");
writeFile(DIR_2, FILE_1);
writeFile(DIR_2, FILE_2);
writeFile(DIR_2, DOT_FILE);
// create and initialize dir2
initializeFile(SFTP_ROOT_DIR + "/" + DIR_2, FILE_1, "dir2 file1");
initializeFile(SFTP_ROOT_DIR + "/" + DIR_2, FILE_2, "dir2 file2");
initializeFile(SFTP_ROOT_DIR + "/" + DIR_2, DOT_FILE, "dir2 foo");
final File linkedDirectory = new File(serverDirectory, LINKED_DIRECTORY);
final File linkedDirectoryTarget = new File(serverDirectory.getAbsolutePath(), DIR_1);
Files.createSymbolicLink(linkedDirectory.toPath(), linkedDirectoryTarget.toPath());
// Create a symbolic link so that dir3/dir1 links to dir1 so we can test following links
final Path targetPath = Paths.get("../" + DIR_1);
final File secondDirectory = new File(serverDirectory, DIR_2);
final File linkedFile = new File(serverDirectory, LINKED_FILE);
final File linkedFileTarget = new File(secondDirectory, FILE_1);
Files.createSymbolicLink(linkedFile.toPath(), linkedFileTarget.toPath());
final String dir3Path = SFTP_ROOT_DIR + "/" + DIR_3;
FileUtils.ensureDirectoryExistAndCanAccess(new File(dir3Path));
final Path linkPath = Paths.get(dir3Path + "/" + DIR_1);
final File emptyDirectory = new File(serverDirectory, EMPTY_DIRECTORY);
assertTrue(emptyDirectory.mkdirs());
Files.createSymbolicLink(linkPath, targetPath);
// create dir4 for writing files
final File dir4File = new File(SFTP_ROOT_DIR + "/" + DIR_4);
FileUtils.ensureDirectoryExistAndCanAccess(dir4File);
startServer();
}
private void initializeFile(final String path, final String filename, final String content) throws IOException {
final File parent = new File(path);
if (!parent.exists()) {
assertTrue("Failed to create parent directory: " + path, parent.mkdirs());
}
final File file = new File(parent, filename);
try (final OutputStream out = new FileOutputStream(file);
final Writer writer = new OutputStreamWriter(out)) {
writer.write(content);
writer.flush();
}
@AfterEach
public void stopServer() throws IOException {
sshServer.stop(true);
}
@Test
@ -143,20 +125,18 @@ public class ITestSFTPTransferWithSSHTestServer {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_2);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
final FileInfo file1Info = listing.stream().filter(f -> f.getFileName().equals(FILE_1)).findFirst().orElse(null);
assertNotNull(file1Info);
assertFalse(file1Info.isDirectory());
assertEquals("rw-r--r--", file1Info.getPermissions());
final FileInfo file2Info = listing.stream().filter(f -> f.getFileName().equals(FILE_2)).findFirst().orElse(null);
assertNotNull(file2Info);
assertFalse(file2Info.isDirectory());
assertEquals("rw-r--r--", file2Info.getPermissions());
}
}
@ -166,8 +146,8 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.REMOTE_PATH, DIR_2);
properties.put(SFTPTransfer.IGNORE_DOTTED_FILES, "false");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(3, listing.size());
@ -182,8 +162,8 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.REMOTE_PATH, DIR_1);
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "false");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(0, listing.size());
}
@ -195,8 +175,8 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.REMOTE_PATH, DIR_1);
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "true");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(4, listing.size());
}
@ -205,28 +185,26 @@ public class ITestSFTPTransferWithSSHTestServer {
@Test
public void testGetListingWithoutSymlinks() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_3);
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "true");
properties.put(SFTPTransfer.FOLLOW_SYMLINK, "false");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(0, listing.size());
assertEquals(6, listing.size());
}
}
@Test
public void testGetListingWithSymlinks() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_3);
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "true");
properties.put(SFTPTransfer.FOLLOW_SYMLINK, "true");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(4, listing.size());
assertEquals(11, listing.size());
}
}
@ -237,8 +215,8 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "true");
// first listing is without batch size and shows 4 results
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(4, listing.size());
}
@ -246,8 +224,8 @@ public class ITestSFTPTransferWithSSHTestServer {
// set a batch size of 2 and ensure we get 2 results
properties.put(SFTPTransfer.REMOTE_POLL_BATCH_SIZE, "2");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
}
@ -262,8 +240,8 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "true");
properties.put(SFTPTransfer.FILE_FILTER_REGEX, fileFilterRegex);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
@ -281,32 +259,21 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "true");
properties.put(SFTPTransfer.PATH_FILTER_REGEX, pathFilterRegex);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
// a listing will have fullPathFileName like "./dir1/child1/file1.txt" so to verify the path pattern
// we need to remove the file part and relativize based on the remote path to get "dir1/child1"
listing.forEach(f -> {
final String filename = f.getFileName();
final String path = f.getFullPathFileName().replace(filename, "");
final Path fullPath = Paths.get(path);
final Path relPath = Paths.get(remotePath).relativize(fullPath);
assertTrue(relPath.toString().matches(pathFilterRegex));
});
}
}
@Test(expected = FileNotFoundException.class)
@Test
public void testGetListingWhenRemotePathDoesNotExist() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, "DOES-NOT-EXIST");
properties.put(SFTPTransfer.RECURSIVE_SEARCH, "true");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
transfer.getListing(true);
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
assertThrows(FileNotFoundException.class, () -> transfer.getListing(FILTERING_ENABLED));
}
}
@ -315,9 +282,9 @@ public class ITestSFTPTransferWithSSHTestServer {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_2);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory has two files
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
@ -327,7 +294,7 @@ public class ITestSFTPTransferWithSSHTestServer {
}
// verify there are now zero files
final List<FileInfo> listingAfterDelete = transfer.getListing(true);
final List<FileInfo> listingAfterDelete = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listingAfterDelete);
assertEquals(0, listingAfterDelete.size());
}
@ -338,9 +305,9 @@ public class ITestSFTPTransferWithSSHTestServer {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_2);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory has two files
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
@ -352,50 +319,44 @@ public class ITestSFTPTransferWithSSHTestServer {
}
// verify there are now zero files
final List<FileInfo> listingAfterDelete = transfer.getListing(true);
final List<FileInfo> listingAfterDelete = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listingAfterDelete);
assertEquals(0, listingAfterDelete.size());
}
}
@Test(expected = FileNotFoundException.class)
@Test
public void testDeleteFileWhenDoesNotExist() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
transfer.deleteFile(null, null, "foo/bar/does-not-exist.txt");
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
assertThrows(FileNotFoundException.class, () -> transfer.deleteFile(null, null, "foo/bar/does-not-exist.txt"));
}
}
@Test
public void testDeleteDirectory() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_4);
properties.put(SFTPTransfer.REMOTE_PATH, EMPTY_DIRECTORY);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory exists
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(0, listing.size());
transfer.deleteDirectory(null, DIR_4);
transfer.deleteDirectory(null, EMPTY_DIRECTORY);
// verify the directory no longer exists
try {
transfer.getListing(true);
Assert.fail("Should have thrown exception");
} catch (FileNotFoundException e) {
// nothing to do, expected
}
assertThrows(FileNotFoundException.class, () -> transfer.getListing(FILTERING_ENABLED));
}
}
@Test(expected = IOException.class)
@Test
public void testDeleteDirectoryWhenDoesNotExist() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
transfer.deleteDirectory(null, "DOES-NOT-EXIST");
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
assertThrows(IOException.class, () -> transfer.deleteDirectory(null, "DOES-NOT-EXIST"));
}
}
@ -405,20 +366,15 @@ public class ITestSFTPTransferWithSSHTestServer {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, remotePath);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory does not exist
try {
transfer.getListing(true);
Assert.fail("Should have failed");
} catch (FileNotFoundException e) {
// Nothing to do, expected
}
assertThrows(FileNotFoundException.class, () -> transfer.getListing(FILTERING_ENABLED));
final String absolutePath = transfer.getAbsolutePath(null, remotePath);
transfer.ensureDirectoryExists(null, new File(absolutePath));
// verify the directory now exists
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(0, listing.size());
}
@ -430,20 +386,14 @@ public class ITestSFTPTransferWithSSHTestServer {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, remotePath);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory does not exist
try {
transfer.getListing(true);
Assert.fail("Should have failed");
} catch (FileNotFoundException e) {
// Nothing to do, expected
}
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
assertThrows(FileNotFoundException.class, () -> transfer.getListing(FILTERING_ENABLED));
final String absolutePath = transfer.getAbsolutePath(null, remotePath);
transfer.ensureDirectoryExists(null, new File(absolutePath));
// verify the directory now exists
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(0, listing.size());
}
@ -454,9 +404,9 @@ public class ITestSFTPTransferWithSSHTestServer {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_2);
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory already exists
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
@ -472,39 +422,33 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.REMOTE_PATH, remotePath);
properties.put(SFTPTransfer.DISABLE_DIRECTORY_LISTING, "true");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory does not exist
try {
transfer.getListing(true);
Assert.fail("Should have failed");
} catch (FileNotFoundException e) {
// Nothing to do, expected
}
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
assertThrows(FileNotFoundException.class, () -> transfer.getListing(FILTERING_ENABLED));
final String absolutePath = transfer.getAbsolutePath(null, remotePath);
transfer.ensureDirectoryExists(null, new File(absolutePath));
// verify the directory now exists
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(0, listing.size());
}
}
@Test(expected = IOException.class)
@Test
public void testEnsureDirectoryExistsWithDirectoryListingDisabledAndAlreadyExists() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
properties.put(SFTPTransfer.REMOTE_PATH, DIR_2);
properties.put(SFTPTransfer.DISABLE_DIRECTORY_LISTING, "true");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory already exists
final List<FileInfo> listing = transfer.getListing(true);
final List<FileInfo> listing = transfer.getListing(FILTERING_ENABLED);
assertNotNull(listing);
assertEquals(2, listing.size());
final String absolutePath = transfer.getAbsolutePath(null, DIR_2);
transfer.ensureDirectoryExists(null, new File(absolutePath));
assertThrows(IOException.class, () -> transfer.ensureDirectoryExists(null, new File(absolutePath)));
}
}
@ -516,14 +460,8 @@ public class ITestSFTPTransferWithSSHTestServer {
properties.put(SFTPTransfer.REMOTE_PATH, remotePath);
properties.put(SFTPTransfer.DISABLE_DIRECTORY_LISTING, "true");
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
// verify the directory does not exist
try {
transfer.getListing(true);
Assert.fail("Should have failed");
} catch (FileNotFoundException e) {
// Nothing to do, expected
}
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
assertThrows(FileNotFoundException.class, () -> transfer.getListing(FILTERING_ENABLED));
// Should swallow exception here
final String absolutePath = transfer.getAbsolutePath(null, remotePath);
@ -535,7 +473,7 @@ public class ITestSFTPTransferWithSSHTestServer {
public void testGetRemoteFileInfo() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final FileInfo fileInfo = transfer.getRemoteFileInfo(null, DIR_2, FILE_1);
assertNotNull(fileInfo);
assertEquals(FILE_1, fileInfo.getFileName());
@ -546,7 +484,7 @@ public class ITestSFTPTransferWithSSHTestServer {
public void testGetRemoteFileInfoWhenPathDoesNotExist() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final FileInfo fileInfo = transfer.getRemoteFileInfo(null, "DOES-NOT-EXIST", FILE_1);
assertNull(fileInfo);
}
@ -556,7 +494,7 @@ public class ITestSFTPTransferWithSSHTestServer {
public void testGetRemoteFileInfoWhenFileDoesNotExist() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final FileInfo fileInfo = transfer.getRemoteFileInfo(null, DIR_2, "DOES-NOT-EXIST");
assertNull(fileInfo);
}
@ -566,7 +504,7 @@ public class ITestSFTPTransferWithSSHTestServer {
public void testGetRemoteFileInfoWhenFileIsADirectory() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final FileInfo fileInfo = transfer.getRemoteFileInfo(null, DIR_1, DIR_1_CHILD_1);
assertNull(fileInfo);
}
@ -576,7 +514,7 @@ public class ITestSFTPTransferWithSSHTestServer {
public void testRename() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final String source = DIR_2 + "/" + FILE_1;
final String target = DIR_2 + "/" + FILE_1 + "-RENAMED";
@ -590,29 +528,29 @@ public class ITestSFTPTransferWithSSHTestServer {
}
}
@Test(expected = FileNotFoundException.class)
@Test
public void testRenameWhenSourceDoesNotExist() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final String source = DIR_2 + "/DOES-NOT-EXIST";
final String target = DIR_2 + "/" + FILE_1 + "-RENAMED";
transfer.rename(null, source, target);
assertThrows(FileNotFoundException.class, () -> transfer.rename(null, source, target));
}
}
@Test(expected = IOException.class)
@Test
public void testRenameWhenTargetAlreadyExists() throws IOException {
final Map<PropertyDescriptor, String> properties = createBaseProperties();
try(final SFTPTransfer transfer = createSFTPTransfer(properties)) {
try (final SFTPTransfer transfer = createSFTPTransfer(properties)) {
final String source = DIR_2 + "/" + FILE_1;
final String target = DIR_2 + "/" + FILE_2;
final FileInfo targetInfoBefore = transfer.getRemoteFileInfo(null, DIR_2, FILE_2);
assertNotNull(targetInfoBefore);
transfer.rename(null, source, target);
assertThrows(IOException.class, () -> transfer.rename(null, source, target));
}
}
@ -626,23 +564,23 @@ public class ITestSFTPTransferWithSSHTestServer {
final String filename = "test-put-simple.txt";
final String fileContent = "this is a test";
try(final SFTPTransfer transfer = createSFTPTransfer(properties);
try (final SFTPTransfer transfer = createSFTPTransfer(properties);
final InputStream in = new ByteArrayInputStream(fileContent.getBytes(StandardCharsets.UTF_8))) {
// Verify file does not already exist
final FileInfo fileInfoBefore = transfer.getRemoteFileInfo(null, DIR_4, filename);
final FileInfo fileInfoBefore = transfer.getRemoteFileInfo(null, EMPTY_DIRECTORY, filename);
assertNull(fileInfoBefore);
final String fullPath = transfer.put(null, DIR_4, filename, in);
final String fullPath = transfer.put(null, EMPTY_DIRECTORY, filename, in);
assertNotNull(fullPath);
// Verify file now exists
final FileInfo fileInfoAfter = transfer.getRemoteFileInfo(null, DIR_4, filename);
final FileInfo fileInfoAfter = transfer.getRemoteFileInfo(null, EMPTY_DIRECTORY, filename);
assertNotNull(fileInfoAfter);
assertEquals(permissions, fileInfoAfter.getPermissions());
// Verify correct content was written
final File writtenFile = new File(SFTP_ROOT_DIR + "/" + DIR_4 + "/" + filename);
final File writtenFile = new File(serverDirectory, EMPTY_DIRECTORY + "/" + filename);
final String retrievedContent = IOUtils.toString(writtenFile.toURI(), StandardCharsets.UTF_8);
assertEquals(fileContent, retrievedContent);
}
@ -659,18 +597,18 @@ public class ITestSFTPTransferWithSSHTestServer {
final String filename = "test-put-simple.txt";
final String fileContent = "this is a test";
try(final SFTPTransfer transfer = createSFTPTransfer(properties);
try (final SFTPTransfer transfer = createSFTPTransfer(properties);
final InputStream in = new ByteArrayInputStream(fileContent.getBytes(StandardCharsets.UTF_8))) {
// Verify file does not already exist
final FileInfo fileInfoBefore = transfer.getRemoteFileInfo(null, DIR_4, filename);
final FileInfo fileInfoBefore = transfer.getRemoteFileInfo(null, EMPTY_DIRECTORY, filename);
assertNull(fileInfoBefore);
final String fullPath = transfer.put(null, DIR_4, filename, in);
final String fullPath = transfer.put(null, EMPTY_DIRECTORY, filename, in);
assertNotNull(fullPath);
// Verify file now exists
final FileInfo fileInfoAfter = transfer.getRemoteFileInfo(null, DIR_4, filename);
final FileInfo fileInfoAfter = transfer.getRemoteFileInfo(null, EMPTY_DIRECTORY, filename);
assertNotNull(fileInfoAfter);
assertEquals(permissions, fileInfoAfter.getPermissions());
}
@ -691,25 +629,25 @@ public class ITestSFTPTransferWithSSHTestServer {
final String filename = "test-put-simple.txt";
final String fileContent = "this is a test";
try(final SFTPTransfer transfer = createSFTPTransfer(properties);
try (final SFTPTransfer transfer = createSFTPTransfer(properties);
final InputStream in = new ByteArrayInputStream(fileContent.getBytes(StandardCharsets.UTF_8))) {
// Verify file does not already exist
final FileInfo fileInfoBefore = transfer.getRemoteFileInfo(null, DIR_4, filename);
final FileInfo fileInfoBefore = transfer.getRemoteFileInfo(null, EMPTY_DIRECTORY, filename);
assertNull(fileInfoBefore);
final String fullPath = transfer.put(null, DIR_4, filename, in);
final String fullPath = transfer.put(null, EMPTY_DIRECTORY, filename, in);
assertNotNull(fullPath);
// Verify file now exists
final FileInfo fileInfoAfter = transfer.getRemoteFileInfo(null, DIR_4, filename);
final FileInfo fileInfoAfter = transfer.getRemoteFileInfo(null, EMPTY_DIRECTORY, filename);
assertNotNull(fileInfoAfter);
assertEquals(permissions, fileInfoAfter.getPermissions());
assertEquals(expectedLastModifiedTime, fileInfoAfter.getLastModifiedTime());
}
}
@Test(expected = IOException.class)
@Test
public void testPutWhenFileAlreadyExists() throws IOException {
final String permissions = "rw-rw-rw-";
@ -718,7 +656,7 @@ public class ITestSFTPTransferWithSSHTestServer {
final String fileContent = "this is a test";
try(final SFTPTransfer transfer = createSFTPTransfer(properties);
try (final SFTPTransfer transfer = createSFTPTransfer(properties);
final InputStream in = new ByteArrayInputStream(fileContent.getBytes(StandardCharsets.UTF_8))) {
// Verify file already exists
@ -726,11 +664,11 @@ public class ITestSFTPTransferWithSSHTestServer {
assertNotNull(fileInfoBefore);
// Should fail because file already exists
transfer.put(null, DIR_2, FILE_1, in);
assertThrows(IOException.class, () -> transfer.put(null, DIR_2, FILE_1, in));
}
}
@Test(expected = IOException.class)
@Test
public void testPutWhenDirectoryDoesNotExist() throws IOException {
final String permissions = "rw-rw-rw-";
@ -739,19 +677,19 @@ public class ITestSFTPTransferWithSSHTestServer {
final String fileContent = "this is a test";
try(final SFTPTransfer transfer = createSFTPTransfer(properties);
try (final SFTPTransfer transfer = createSFTPTransfer(properties);
final InputStream in = new ByteArrayInputStream(fileContent.getBytes(StandardCharsets.UTF_8))) {
transfer.put(null, "DOES-NOT-EXIST", FILE_1, in);
assertThrows(IOException.class, () -> transfer.put(null, "DOES-NOT-EXIST", FILE_1, in));
}
}
private Map<PropertyDescriptor, String> createBaseProperties() {
final Map<PropertyDescriptor,String> properties = new HashMap<>();
properties.put(SFTPTransfer.HOSTNAME, "localhost");
properties.put(SFTPTransfer.PORT, Integer.toString(sshTestServer.getSSHPort()));
properties.put(SFTPTransfer.USERNAME, sshTestServer.getUsername());
properties.put(SFTPTransfer.PASSWORD, sshTestServer.getPassword());
properties.put(SFTPTransfer.STRICT_HOST_KEY_CHECKING, "false");
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(SFTPTransfer.HOSTNAME, LOCALHOST);
properties.put(SFTPTransfer.PORT, Integer.toString(sshServer.getPort()));
properties.put(SFTPTransfer.USERNAME, USERNAME);
properties.put(SFTPTransfer.PASSWORD, PASSWORD);
properties.put(SFTPTransfer.STRICT_HOST_KEY_CHECKING, Boolean.FALSE.toString());
return properties;
}
@ -760,4 +698,22 @@ public class ITestSFTPTransferWithSSHTestServer {
final ComponentLog logger = Mockito.mock(ComponentLog.class);
return new SFTPTransfer(propertyContext, logger);
}
private void startServer() throws IOException {
sshServer = SshServer.setUpDefaultServer();
sshServer.setHost(LOCALHOST);
sshServer.setPasswordAuthenticator((username, password, serverSession) -> USERNAME.equals(username) && PASSWORD.equals(password));
sshServer.setKeyPairProvider(new SimpleGeneratorHostKeyProvider());
sshServer.setFileSystemFactory(new VirtualFileSystemFactory(serverDirectory.toPath()));
sshServer.setSubsystemFactories(Collections.singletonList(new SftpSubsystemFactory()));
sshServer.start();
}
private void writeFile(final String... pathElements) throws IOException {
final Path path = Paths.get(serverDirectory.getAbsolutePath(), pathElements);
final File parentFile = path.toFile().getParentFile();
FileUtils.forceMkdir(parentFile);
final byte[] contents = path.toFile().getAbsolutePath().getBytes(StandardCharsets.UTF_8);
Files.write(path, contents);
}
}