diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java index 3beab658ad..9ea793daf9 100644 --- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java +++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java @@ -121,13 +121,13 @@ public class IndexConfiguration { } } - public File getWritableIndexDirectory(final File provenanceLogFile) { + public File getWritableIndexDirectory(final File provenanceLogFile, final long newIndexTimestamp) { lock.lock(); try { final File storageDirectory = provenanceLogFile.getParentFile(); List indexDirectories = this.indexDirectoryMap.get(storageDirectory); if (indexDirectories == null) { - final File newDir = addNewIndex(storageDirectory, provenanceLogFile); + final File newDir = addNewIndex(storageDirectory, provenanceLogFile, newIndexTimestamp); indexDirectories = new ArrayList<>(); indexDirectories.add(newDir); indexDirectoryMap.put(storageDirectory, indexDirectories); @@ -135,7 +135,7 @@ public class IndexConfiguration { } if (indexDirectories.isEmpty()) { - final File newDir = addNewIndex(storageDirectory, provenanceLogFile); + final File newDir = addNewIndex(storageDirectory, provenanceLogFile, newIndexTimestamp); indexDirectories.add(newDir); return newDir; } @@ -143,7 +143,7 @@ public class IndexConfiguration { final File lastDir = indexDirectories.get(indexDirectories.size() - 1); final long size = getSize(lastDir); if (size > repoConfig.getDesiredIndexSize()) { - final File newDir = addNewIndex(storageDirectory, provenanceLogFile); + final File newDir = addNewIndex(storageDirectory, provenanceLogFile, newIndexTimestamp); indexDirectories.add(newDir); return newDir; } else { @@ -154,14 +154,14 @@ public class IndexConfiguration { } } - private File addNewIndex(final File storageDirectory, final File provenanceLogFile) { + private File addNewIndex(final File storageDirectory, final File provenanceLogFile, final long newIndexTimestamp) { // Build the event time of the first record into the index's filename so that we can determine // which index files to look at when we perform a search. We use the timestamp of the first record // in the Provenance Log file, rather than the current time, because we may perform the Indexing // retroactively. Long firstEntryTime = getFirstEntryTime(provenanceLogFile); if (firstEntryTime == null) { - firstEntryTime = System.currentTimeMillis(); + firstEntryTime = newIndexTimestamp; } return new File(storageDirectory, "index-" + firstEntryTime); } @@ -222,7 +222,7 @@ public class IndexConfiguration { } }); - for (File indexDir : sortedIndexDirectories) { + for (final File indexDir : sortedIndexDirectories) { // If the index was last modified before the start time, we know that it doesn't // contain any data for us to query. if (startTime != null && indexDir.lastModified() < startTime) { @@ -282,7 +282,7 @@ public class IndexConfiguration { } boolean foundIndexCreatedLater = false; - for (File indexDir : sortedIndexDirectories) { + for (final File indexDir : sortedIndexDirectories) { // If the index was last modified before the log file was created, we know the index doesn't include // any data for the provenance log. if (indexDir.lastModified() < firstEntryTime) { diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java index 214fc7cc17..3bdd38f5b2 100644 --- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java +++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java @@ -1231,6 +1231,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository } }); + long earliestTimestamp = System.currentTimeMillis(); for (final RecordReader reader : readers) { StandardProvenanceEventRecord record = null; @@ -1252,6 +1253,9 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository continue; } + if ( record.getEventTime() < earliestTimestamp ) { + earliestTimestamp = record.getEventTime(); + } recordToReaderMap.put(record, reader); } @@ -1262,7 +1266,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository final IndexingAction indexingAction = new IndexingAction(this); - final File indexingDirectory = indexConfig.getWritableIndexDirectory(writerFile); + final File indexingDirectory = indexConfig.getWritableIndexDirectory(writerFile, earliestTimestamp); final IndexWriter indexWriter = indexManager.borrowIndexWriter(indexingDirectory); try { long maxId = 0L; diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java index 5e4f69d509..502068bea5 100644 --- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java +++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/LineageQuery.java @@ -96,8 +96,8 @@ public class LineageQuery { final DocsReader docsReader = new DocsReader(repo.getConfiguration().getStorageDirectories()); final Set recs = docsReader.read(uuidQueryTopDocs, searcher.getIndexReader(), repo.getAllLogFiles(), new AtomicInteger(0), Integer.MAX_VALUE); final long readDocsEnd = System.nanoTime(); - logger.debug("Finished Lineage Query; Lucene search took {} millis, reading records took {} millis", - TimeUnit.NANOSECONDS.toMillis(searchEnd - searchStart), TimeUnit.NANOSECONDS.toMillis(readDocsEnd - searchEnd)); + logger.debug("Finished Lineage Query against {}; Lucene search took {} millis, reading records took {} millis", + indexDirectory, TimeUnit.NANOSECONDS.toMillis(searchEnd - searchStart), TimeUnit.NANOSECONDS.toMillis(readDocsEnd - searchEnd)); return recs; } finally {