Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-6405 GetHDFSFileInfo ignores files in the root/start directory 

modifying test cases, fixing file processing.

Testing done:
- Unit tests
- Manual run against a hadoop instance

This closes #3558
This commit is contained in:
Ferenc Szabó 2019-06-28 10:28:12 +02:00 committed by Jeff Storck
parent f1ae05974e
commit 3d21059e95
5 changed files with 44 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/main/java/org/apache/nifi/processors/hadoop/GetHDFSFileInfo.java
View File

@ -369,8 +369,7 @@ public class GetHDFSFileInfo extends AbstractHadoopProcessor {
// Decide what to do with child: if requested FF per object or per dir - just emit new FF with info in 'o' object
if (vo != null && !statsOnly) {
parent.addChild(vo);
if (p != null && req.isRecursive
&& vo.isFile() && !vo.isSymlink()) {
if (vo.isFile() && !vo.isSymlink()) {
processHDFSObject(context, session, origFF, req, vo, false);
}
}

49
nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/java/org/apache/nifi/processors/hadoop/TestGetHDFSFileInfo.java
View File

@ -302,8 +302,8 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified", ""+1523456000000L);
mff.assertAttributeEquals("hdfs.length", ""+500);
mff.assertAttributeEquals("hdfs.count.files", ""+5);
mff.assertAttributeEquals("hdfs.length", ""+900);
mff.assertAttributeEquals("hdfs.count.files", ""+9);
mff.assertAttributeEquals("hdfs.count.dirs", ""+10);
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rwxr-xr-x");
@ -329,7 +329,7 @@ public class TestGetHDFSFileInfo {
runner.run();
runner.assertTransferCount(GetHDFSFileInfo.REL_ORIGINAL, 0);
runner.assertTransferCount(GetHDFSFileInfo.REL_SUCCESS, 7);
runner.assertTransferCount(GetHDFSFileInfo.REL_SUCCESS, 9);
runner.assertTransferCount(GetHDFSFileInfo.REL_FAILURE, 0);
runner.assertTransferCount(GetHDFSFileInfo.REL_NOT_FOUND, 0);
@ -342,8 +342,8 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified", ""+1523456000000L);
mff.assertAttributeEquals("hdfs.length", ""+500);
mff.assertAttributeEquals("hdfs.count.files", ""+5);
mff.assertAttributeEquals("hdfs.length", ""+900);
mff.assertAttributeEquals("hdfs.count.files", ""+9);
mff.assertAttributeEquals("hdfs.count.dirs", ""+10);
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rwxr-xr-x");
@ -426,11 +426,37 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rw-r--r--");
mff.assertAttributeNotExists("hdfs.status");
}else if (mff.getAttribute("hdfs.objectName").equals("regFile4")) {
matchCount++;
mff.assertAttributeEquals("hdfs.path", "/some/home/mydir");
mff.assertAttributeEquals("hdfs.type", "file");
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified", ""+1523456000000L);
mff.assertAttributeEquals("hdfs.length", ""+100);
mff.assertAttributeNotExists("hdfs.count.files");
mff.assertAttributeNotExists("hdfs.count.dirs");
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rw-r--r--");
mff.assertAttributeNotExists("hdfs.status");
}else if (mff.getAttribute("hdfs.objectName").equals("regFile5")) {
matchCount++;
mff.assertAttributeEquals("hdfs.path", "/some/home/mydir");
mff.assertAttributeEquals("hdfs.type", "file");
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified", ""+1523456000000L);
mff.assertAttributeEquals("hdfs.length", ""+100);
mff.assertAttributeNotExists("hdfs.count.files");
mff.assertAttributeNotExists("hdfs.count.dirs");
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rw-r--r--");
mff.assertAttributeNotExists("hdfs.status");
}else {
runner.assertNotValid();
}
}
Assert.assertEquals(matchCount, 7);
Assert.assertEquals(matchCount, 9);
}
@Test
@ -462,8 +488,8 @@ public class TestGetHDFSFileInfo {
mff.assertAttributeEquals("hdfs.owner", "owner");
mff.assertAttributeEquals("hdfs.group", "group");
mff.assertAttributeEquals("hdfs.lastModified", ""+1523456000000L);
mff.assertAttributeEquals("hdfs.length", ""+500);
mff.assertAttributeEquals("hdfs.count.files", ""+5);
mff.assertAttributeEquals("hdfs.length", ""+900);
mff.assertAttributeEquals("hdfs.count.files", ""+9);
mff.assertAttributeEquals("hdfs.count.dirs", ""+10);
mff.assertAttributeEquals("hdfs.replication", ""+3);
mff.assertAttributeEquals("hdfs.permissions", "rwxr-xr-x");
@ -540,9 +566,14 @@ public class TestGetHDFSFileInfo {
/*
* For all basic tests, this provides a structure of files in dirs:
* Total number of dirs: 9 (1 root, 4 dotted)
* Total number of files: 4 (2 dotted)
* Total number of files: 8 (4 dotted)
*/
protected void setFileSystemBasicTree(final MockFileSystem fs) {
fs.addFileStatus(fs.newDir("/some/home/mydir"), fs.newFile("/some/home/mydir/regFile4"));
fs.addFileStatus(fs.newDir("/some/home/mydir"), fs.newFile("/some/home/mydir/regFile5"));
fs.addFileStatus(fs.newDir("/some/home/mydir"), fs.newFile("/some/home/mydir/.dotFile4"));
fs.addFileStatus(fs.newDir("/some/home/mydir"), fs.newFile("/some/home/mydir/.dotFile5"));
fs.addFileStatus(fs.newDir("/some/home/mydir"), fs.newDir("/some/home/mydir/dir1"));
fs.addFileStatus(fs.newDir("/some/home/mydir"), fs.newDir("/some/home/mydir/dir2"));
fs.addFileStatus(fs.newDir("/some/home/mydir"), fs.newDir("/some/home/mydir/.dir3"));

2
nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupAllToAttributes.json
View File

@ -1 +1 @@
{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"500","replication":"3","countFiles":"5","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x","content":[{"objectName":"regFile","path":"/some/home/mydir/dir1","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regDir","path":"/some/home/mydir/dir1","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"}]},{"objectName":"dir2","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x","content":[{"objectName":"regDir2","path":"/some/home/mydir/dir2","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"},{"objectName":"regFile2","path":"/some/home/mydir/dir2","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"}]}]}
{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"900","replication":"3","countFiles":"9","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"regFile5","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regFile4","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x","content":[{"objectName":"regFile","path":"/some/home/mydir/dir1","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regDir","path":"/some/home/mydir/dir1","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"}]},{"objectName":"dir2","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x","content":[{"objectName":"regDir2","path":"/some/home/mydir/dir2","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"},{"objectName":"regFile2","path":"/some/home/mydir/dir2","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"}]}]}

2
nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRecursiveGroupDirToAttributes-mydir.json
View File

@ -1 +1 @@
{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"500","replication":"3","countFiles":"5","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x"},{"objectName":"dir2","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x"}]}
{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"900","replication":"3","countFiles":"9","countDirs":"10","permissions":"rwxr-xr-x","content":[{"objectName":"regFile5","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regFile4","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x"},{"objectName":"dir2","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x"}]}

2
nifi-nar-bundles/nifi-hadoop-bundle/nifi-hdfs-processors/src/test/resources/TestGetHDFSFileInfo/testRunWithPermissionsExceptionContent.json
View File

@ -1 +1 @@
{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"500","replication":"3","countFiles":"5","countDirs":"11","permissions":"rwxr-xr-x","content":[{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"4","permissions":"rwxr-xr-x","content":[{"objectName":"list_exception_java.io.InterruptedIOException","path":"/some/home/mydir/dir1","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x","status":"Couldn't list directory: java.io.InterruptedIOException"},{"objectName":"regFile","path":"/some/home/mydir/dir1","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regDir","path":"/some/home/mydir/dir1","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"}]},{"objectName":"dir2","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x","content":[{"objectName":"regDir2","path":"/some/home/mydir/dir2","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"},{"objectName":"regFile2","path":"/some/home/mydir/dir2","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"}]}]}
{"objectName":"mydir","path":"/some/home","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"900","replication":"3","countFiles":"9","countDirs":"11","permissions":"rwxr-xr-x","content":[{"objectName":"regFile5","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regFile4","path":"/some/home/mydir","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"dir1","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"4","permissions":"rwxr-xr-x","content":[{"objectName":"list_exception_java.io.InterruptedIOException","path":"/some/home/mydir/dir1","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x","status":"Couldn't list directory: java.io.InterruptedIOException"},{"objectName":"regFile","path":"/some/home/mydir/dir1","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"},{"objectName":"regDir","path":"/some/home/mydir/dir1","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"}]},{"objectName":"dir2","path":"/some/home/mydir","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"200","replication":"3","countFiles":"2","countDirs":"3","permissions":"rwxr-xr-x","content":[{"objectName":"regDir2","path":"/some/home/mydir/dir2","type":"directory","owner":"owner","group":"group","lastModified":"1523456000000","length":"0","replication":"3","countFiles":"0","countDirs":"1","permissions":"rwxr-xr-x"},{"objectName":"regFile2","path":"/some/home/mydir/dir2","type":"file","owner":"owner","group":"group","lastModified":"1523456000000","length":"100","replication":"3","permissions":"rw-r--r--"}]}]}