Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-03-06 01:19:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-5560 This closes #3000. Added Follow SYMLINK support for listFTP & listSFTP and getFTP & getSFTP processors

Browse Source 
Signed-off-by: joewitt <joewitt@apache.org>
This commit is contained in:
Kumara M S Hemantha 2018-09-13 02:19:32 +05:30 committed by joewitt
parent 286a1c880a
commit 71499f774d
No known key found for this signature in database
GPG Key ID: 9093BF854F811A1A
7 changed files with 23 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetFTP.java
View File

@ -72,6 +72,7 @@ public class GetFTP extends GetFileTransfer {
properties.add(FTPTransfer.PATH_FILTER_REGEX);
properties.add(FTPTransfer.POLLING_INTERVAL);
properties.add(FTPTransfer.RECURSIVE_SEARCH);
properties.add(FTPTransfer.FOLLOW_SYMLINK);
properties.add(FTPTransfer.IGNORE_DOTTED_FILES);
properties.add(FTPTransfer.DELETE_ORIGINAL);
properties.add(FTPTransfer.CONNECTION_TIMEOUT);

1
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
View File

@ -71,6 +71,7 @@ public class GetSFTP extends GetFileTransfer {
properties.add(SFTPTransfer.PATH_FILTER_REGEX);
properties.add(SFTPTransfer.POLLING_INTERVAL);
properties.add(SFTPTransfer.RECURSIVE_SEARCH);
properties.add(SFTPTransfer.FOLLOW_SYMLINK);
properties.add(SFTPTransfer.IGNORE_DOTTED_FILES);
properties.add(SFTPTransfer.DELETE_ORIGINAL);
properties.add(SFTPTransfer.CONNECTION_TIMEOUT);

1
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFTP.java
View File

@ -79,6 +79,7 @@ public class ListFTP extends ListFileTransfer {
properties.add(REMOTE_PATH);
properties.add(DISTRIBUTED_CACHE_SERVICE);
properties.add(FTPTransfer.RECURSIVE_SEARCH);
properties.add(FTPTransfer.FOLLOW_SYMLINK);
properties.add(FTPTransfer.FILE_FILTER_REGEX);
properties.add(FTPTransfer.PATH_FILTER_REGEX);
properties.add(FTPTransfer.IGNORE_DOTTED_FILES);

1
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
View File

@ -83,6 +83,7 @@ public class ListSFTP extends ListFileTransfer {
properties.add(REMOTE_PATH);
properties.add(DISTRIBUTED_CACHE_SERVICE);
properties.add(SFTPTransfer.RECURSIVE_SEARCH);
properties.add(SFTPTransfer.FOLLOW_SYMLINK);
properties.add(SFTPTransfer.FILE_FILTER_REGEX);
properties.add(SFTPTransfer.PATH_FILTER_REGEX);
properties.add(SFTPTransfer.IGNORE_DOTTED_FILES);

7
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java
View File

@ -204,6 +204,7 @@ public class FTPTransfer implements FileTransfer {
final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean();
final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean();
final boolean symlink = ctx.getProperty(FileTransfer.FOLLOW_SYMLINK).asBoolean();
final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue();
final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex);
final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue();
@ -255,11 +256,13 @@ public class FTPTransfer implements FileTransfer {
final File newFullPath = new File(path, filename);
final String newFullForwardPath = newFullPath.getPath().replace("\\", "/");
if (recurse && file.isDirectory()) {
// if is a directory and we're supposed to recurse
// OR if is a link and we're supposed to follow symlink
if ((recurse && file.isDirectory()) || (symlink && file.isSymbolicLink())) {
try {
listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count));
} catch (final IOException e) {
logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory", e);
logger.error("Unable to get listing from " + newFullForwardPath + "; skipping", e);
}
}

8
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileTransfer.java
View File

@ -141,6 +141,14 @@ public interface FileTransfer extends Closeable {
.defaultValue("false")
.allowableValues("true", "false")
.build();
public static final PropertyDescriptor FOLLOW_SYMLINK = new PropertyDescriptor.Builder()
.name("follow-symlink")
.displayName("Follow symlink")
.description("If true, will pull even symbolic files and also nested symbolic subdirectories; otherwise, will not read symbolic files and will not traverse symbolic link subdirectories")
.required(true)
.defaultValue("false")
.allowableValues("true", "false")
.build();
public static final PropertyDescriptor FILE_FILTER_REGEX = new PropertyDescriptor.Builder()
.name("File Filter Regex")
.description("Provides a Java Regular Expression for filtering Filenames; if a filter is supplied, only files whose names match that Regular Expression will be fetched")

8
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
View File

@ -188,6 +188,7 @@ public class SFTPTransfer implements FileTransfer {
final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean();
final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean();
final boolean symlink = ctx.getProperty(FileTransfer.FOLLOW_SYMLINK).asBoolean();
final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue();
final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex);
final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue();
@ -211,6 +212,7 @@ public class SFTPTransfer implements FileTransfer {
final ChannelSftp sftp = getChannel(null);
final boolean isPathMatch = pathFilterMatches;
//subDirs list is used for both 'sub directories' and 'symlinks'
final List<LsEntry> subDirs = new ArrayList<>();
try {
final LsEntrySelector filter = new LsEntrySelector() {
@ -231,7 +233,8 @@ public class SFTPTransfer implements FileTransfer {
}
// if is a directory and we're supposed to recurse
if (recurse && entry.getAttrs().isDir()) {
// OR if is a link and we're supposed to follow symlink
if ((recurse && entry.getAttrs().isDir()) || (symlink && entry.getAttrs().isLink())){
subDirs.add(entry);
return LsEntrySelector.CONTINUE;
}
@ -278,9 +281,10 @@ public class SFTPTransfer implements FileTransfer {
try {
getListing(newFullForwardPath, depth + 1, maxResults, listing);
} catch (final IOException e) {
logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory", e);
logger.error("Unable to get listing from " + newFullForwardPath + "; skipping", e);
}
}
}
private FileInfo newFileInfo(final LsEntry entry, String path) {