Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-5262 - Retrieve file attributes only once in ListFile 

Signed-off-by: Pierre Villard <pierre.villard.fr@gmail.com>

This closes #2754.
This commit is contained in:
Marco Gaido 2018-06-04 13:16:49 +02:00 committed by Pierre Villard
parent c3af53ce65
commit eedf1237a1
2 changed files with 55 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListFile.java
View File

@ -39,9 +39,9 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processor.util.list.AbstractListProcessor;
import org.apache.nifi.processors.standard.util.FileInfo;
import org.apache.nifi.processors.standard.util.FileInfoFilter;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
@ -198,7 +198,7 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
private List<PropertyDescriptor> properties;
private Set<Relationship> relationships;
private final AtomicReference<FileFilter> fileFilterRef = new AtomicReference<>();
private final AtomicReference<FileInfoFilter> fileFilterRef = new AtomicReference<>();
private volatile boolean includeFileAttributes;
@ -346,7 +346,7 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
|| IGNORE_HIDDEN_FILES.equals(property);
}
private List<FileInfo> scanDirectory(final File path, final FileFilter filter, final Boolean recurse,
private List<FileInfo> scanDirectory(final File path, final FileInfoFilter filter, final Boolean recurse,
final Long minTimestamp) throws IOException {
final List<FileInfo> listing = new ArrayList<>();
File[] files = path.listFiles();
@ -357,14 +357,16 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
listing.addAll(scanDirectory(file, filter, true, minTimestamp));
}
} else {
if ((minTimestamp == null || file.lastModified() >= minTimestamp) && filter.accept(file)) {
listing.add(new FileInfo.Builder()
.directory(file.isDirectory())
.filename(file.getName())
.fullPathFileName(file.getAbsolutePath())
.size(file.length())
.lastModifiedTime(file.lastModified())
.build());
FileInfo fileInfo = new FileInfo.Builder()
.directory(false)
.filename(file.getName())
.fullPathFileName(file.getAbsolutePath())
.size(file.length())
.lastModifiedTime(file.lastModified())
.build();
if ((minTimestamp == null || fileInfo.getLastModifiedTime() >= minTimestamp)
&& filter.accept(file, fileInfo)) {
listing.add(fileInfo);
}
}
}
@ -373,7 +375,7 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
return listing;
}
private FileFilter createFileFilter(final ProcessContext context) {
private FileInfoFilter createFileFilter(final ProcessContext context) {
final long minSize = context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue();
final Double maxSize = context.getProperty(MAX_SIZE).asDataSize(DataUnit.B);
final long minAge = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
@ -385,16 +387,16 @@ public class ListFile extends AbstractListProcessor<FileInfo> {
final String pathPatternStr = context.getProperty(PATH_FILTER).getValue();
final Pattern pathPattern = (!recurseDirs || pathPatternStr == null) ? null : Pattern.compile(pathPatternStr);
return new FileFilter() {
return new FileInfoFilter() {
@Override
public boolean accept(final File file) {
if (minSize > file.length()) {
public boolean accept(final File file, final FileInfo info) {
if (minSize > info.getSize()) {
return false;
}
if (maxSize != null && maxSize < file.length()) {
if (maxSize != null && maxSize < info.getSize()) {
return false;
}
final long fileAge = System.currentTimeMillis() - file.lastModified();
final long fileAge = System.currentTimeMillis() - info.getLastModifiedTime();
if (minAge > fileAge) {
return false;
}

36
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/FileInfoFilter.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.standard.util;
import java.io.File;
/**
* A filter for files.
*/
public interface FileInfoFilter {
/**
* Tests whether or not the specified file with its
* <code>{@link org.apache.nifi.processors.standard.util.FileInfo}</code> satisfies the
* requirements of this filter.
*
* @param file The file to be tested
* @param info Attributes and info about the file.
* @return <code>true</code> if and only if the <code>file</code> should be included
*/
boolean accept(File file, FileInfo info);
}