Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-1070: Added detailed debug-level logging about how FileSystemRepository is choosing to expire archived data

This commit is contained in:
Mark Payne 2015-10-26 14:36:03 -04:00
parent f8c3377c84
commit aec32a277c
1 changed files with 18 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java
View File

@ -87,6 +87,8 @@ public class FileSystemRepository implements ContentRepository {
public static final Pattern MAX_ARCHIVE_SIZE_PATTERN = Pattern.compile("\\d{1,2}%"); public static final Pattern MAX_ARCHIVE_SIZE_PATTERN = Pattern.compile("\\d{1,2}%");
private static final Logger LOG = LoggerFactory.getLogger(FileSystemRepository.class); private static final Logger LOG = LoggerFactory.getLogger(FileSystemRepository.class);
private final Logger archiveExpirationLog = LoggerFactory.getLogger(FileSystemRepository.class.getName() + ".archive.expiration");
private final Map<String, Path> containers; private final Map<String, Path> containers;
private final List<String> containerNames; private final List<String> containerNames;
private final AtomicLong index; private final AtomicLong index;
@ -620,7 +622,7 @@ public class FileSystemRepository implements ContentRepository {
final File file = path.toFile(); final File file = path.toFile();
if (!file.delete() && file.exists()) { if (!file.delete() && file.exists()) {
LOG.warn("Unable to delete {} at path {}", new Object[]{claim, path}); LOG.warn("Unable to delete {} at path {}", new Object[] {claim, path});
return false; return false;
} }
@ -1187,6 +1189,7 @@ public class FileSystemRepository implements ContentRepository {
} }
private long destroyExpiredArchives(final String containerName, final Path container) throws IOException { private long destroyExpiredArchives(final String containerName, final Path container) throws IOException {
archiveExpirationLog.debug("Destroying Expired Archives for Container {}", containerName);
final List<ArchiveInfo> notYetExceedingThreshold = new ArrayList<>(); final List<ArchiveInfo> notYetExceedingThreshold = new ArrayList<>();
final long removalTimeThreshold = System.currentTimeMillis() - maxArchiveMillis; final long removalTimeThreshold = System.currentTimeMillis() - maxArchiveMillis;
long oldestArchiveDateFound = System.currentTimeMillis(); long oldestArchiveDateFound = System.currentTimeMillis();
@ -1194,6 +1197,7 @@ public class FileSystemRepository implements ContentRepository {
// determine how much space we must have in order to stop deleting old data // determine how much space we must have in order to stop deleting old data
final Long minRequiredSpace = minUsableContainerBytesForArchive.get(containerName); final Long minRequiredSpace = minUsableContainerBytesForArchive.get(containerName);
if (minRequiredSpace == null) { if (minRequiredSpace == null) {
archiveExpirationLog.debug("Could not determine minimum required space so will not destroy any archived data");
return -1L; return -1L;
} }
@ -1204,6 +1208,7 @@ public class FileSystemRepository implements ContentRepository {
final long startNanos = System.nanoTime(); final long startNanos = System.nanoTime();
final long toFree = minRequiredSpace - usableSpace; final long toFree = minRequiredSpace - usableSpace;
final BlockingQueue<ArchiveInfo> fileQueue = archivedFiles.get(containerName); final BlockingQueue<ArchiveInfo> fileQueue = archivedFiles.get(containerName);
archiveExpirationLog.info("Currently {} bytes free for Container {}; requirement is {} byte free, so need to free {} bytes", usableSpace, containerName, minRequiredSpace, toFree);
ArchiveInfo toDelete; ArchiveInfo toDelete;
int deleteCount = 0; int deleteCount = 0;
@ -1229,9 +1234,12 @@ public class FileSystemRepository implements ContentRepository {
if (freed >= toFree) { if (freed >= toFree) {
// If the last mod time indicates that it should be removed, just continue loop. // If the last mod time indicates that it should be removed, just continue loop.
if (deleteBasedOnTimestamp(fileQueue, removalTimeThreshold)) { if (deleteBasedOnTimestamp(fileQueue, removalTimeThreshold)) {
archiveExpirationLog.debug("Freed enough space ({} bytes freed, needed to free {} bytes) but will continue to expire data based on timestamp", freed, toFree);
continue; continue;
} }
archiveExpirationLog.debug("Freed enough space ({} bytes freed, needed to free {} bytes). Finished expiring data", freed, toFree);
final ArchiveInfo archiveInfo = fileQueue.peek(); final ArchiveInfo archiveInfo = fileQueue.peek();
final long oldestArchiveDate = archiveInfo == null ? System.currentTimeMillis() : getLastModTime(archiveInfo.toPath()); final long oldestArchiveDate = archiveInfo == null ? System.currentTimeMillis() : getLastModTime(archiveInfo.toPath());
@ -1256,6 +1264,7 @@ public class FileSystemRepository implements ContentRepository {
} }
// Go through each container and grab the archived data into a List // Go through each container and grab the archived data into a List
archiveExpirationLog.debug("Searching for more archived data to expire");
final StopWatch stopWatch = new StopWatch(true); final StopWatch stopWatch = new StopWatch(true);
for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) { for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) {
final Path sectionContainer = container.resolve(String.valueOf(i)); final Path sectionContainer = container.resolve(String.valueOf(i));
@ -1312,6 +1321,7 @@ public class FileSystemRepository implements ContentRepository {
final long sortRemainingMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - deleteExpiredMillis; final long sortRemainingMillis = stopWatch.getElapsed(TimeUnit.MILLISECONDS) - deleteExpiredMillis;
// Delete the oldest data // Delete the oldest data
archiveExpirationLog.debug("Deleting data based on timestamp");
final Iterator<ArchiveInfo> itr = notYetExceedingThreshold.iterator(); final Iterator<ArchiveInfo> itr = notYetExceedingThreshold.iterator();
int counter = 0; int counter = 0;
while (itr.hasNext()) { while (itr.hasNext()) {