Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-03-04 00:19:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-13186 Corrected Provenance dependency for lucene-backward-codecs

Browse Source 
- Updated references to lucene-9 indexes and removed legacy old style indexes

This closes #8782

Signed-off-by: David Handermann <exceptionfactory@apache.org>i
This commit is contained in:
Joseph Witt 2024-05-08 17:20:30 -07:00 committed by exceptionfactory
parent 4faad38c16
commit d305a169b3
No known key found for this signature in database
13 changed files with 52 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml
View File

@ -52,13 +52,16 @@
<artifactId>nifi-repository-encryption</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
<scope>runtime</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
@ -68,10 +71,10 @@
<excludes combine.children="append">
<exclude>src/test/resources/lucene-4-prov-repo/0.prov</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/toc/0.toc</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_0.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_0.fdx</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_1.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_1.fdx</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_0.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_0.fdx</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_1.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_1.fdx</exclude>
</excludes>
</configuration>
</plugin>

2
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java
View File

@ -170,7 +170,7 @@ public class IndexConfiguration {
if (firstEntryTime == null) {
firstEntryTime = newIndexTimestamp;
}
return new File(storageDirectory, "lucene-8-index-" + firstEntryTime);
return new File(storageDirectory, "lucene-9-index-" + firstEntryTime);
}
public List<File> getIndexDirectories() {

24
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/IndexDirectoryManager.java
View File

@ -51,13 +51,6 @@ public class IndexDirectoryManager {
private static final Pattern LUCENE_8_AND_LATER_INDEX_PATTERN = Pattern.compile("lucene-\\d+-index-(.*)");
private static final FileFilter LUCENE_8_AND_LATER_INDEX_DIRECTORY_FILTER = f -> LUCENE_8_AND_LATER_INDEX_PATTERN.matcher(f.getName()).matches();
private static final Pattern INDEX_FILENAME_PATTERN = DirectoryUtils.INDEX_DIRECTORY_NAME_PATTERN;
private static final FileFilter ALL_INDEX_FILE_FILTER = f -> INDEX_FILENAME_PATTERN.matcher(f.getName()).matches();
private static final Pattern LUCENE_4_INDEX_PATTERN = Pattern.compile("index-(.*)");
private static final FileFilter LUCENE_4_INDEX_FILE_FILTER = f -> LUCENE_4_INDEX_PATTERN.matcher(f.getName()).matches();
private final RepositoryConfiguration repoConfig;
// guarded by synchronizing on 'this'
@ -82,7 +75,7 @@ public class IndexDirectoryManager {
}
for (final File indexDir : indexDirs) {
final Matcher matcher = INDEX_FILENAME_PATTERN.matcher(indexDir.getName());
final Matcher matcher = LUCENE_8_AND_LATER_INDEX_PATTERN.matcher(indexDir.getName());
if (!matcher.matches()) {
continue;
}
@ -131,19 +124,10 @@ public class IndexDirectoryManager {
}
}
public synchronized List<File> getAllIndexDirectories(final boolean includeLucene4Directories, final boolean includeLaterLuceneDirectories) {
public synchronized List<File> getAllIndexDirectories() {
final List<File> allDirectories = new ArrayList<>();
final FileFilter directoryFilter;
if (includeLucene4Directories && includeLaterLuceneDirectories) {
directoryFilter = ALL_INDEX_FILE_FILTER;
} else if (includeLucene4Directories) {
directoryFilter = LUCENE_4_INDEX_FILE_FILTER;
} else if (includeLaterLuceneDirectories) {
directoryFilter = LUCENE_8_AND_LATER_INDEX_DIRECTORY_FILTER;
} else {
throw new IllegalArgumentException("Cannot list all directoreis but excluded Lucene 4 directories and later directories");
}
final FileFilter directoryFilter = LUCENE_8_AND_LATER_INDEX_DIRECTORY_FILTER;
for (final File storageDir : repoConfig.getStorageDirectories().values()) {
final File[] indexDirs = storageDir.listFiles(directoryFilter);
@ -410,7 +394,7 @@ public class IndexDirectoryManager {
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Invalid Partition: " + partitionName));
final File indexDir = new File(storageDir, "lucene-8-index-" + earliestTimestamp);
final File indexDir = new File(storageDir, "lucene-9-index-" + earliestTimestamp);
return indexDir;
}

31
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java
View File

@ -173,7 +173,7 @@ public class LuceneEventIndex implements EventIndex {
private void triggerReindexOfDefunctIndices() {
final ExecutorService rebuildIndexExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Rebuild Defunct Provenance Indices", true));
final List<File> allIndexDirectories = directoryManager.getAllIndexDirectories(true, true);
final List<File> allIndexDirectories = directoryManager.getAllIndexDirectories();
allIndexDirectories.sort(DirectoryUtils.OLDEST_INDEX_FIRST);
final List<File> defunctIndices = detectDefunctIndices(allIndexDirectories);
@ -182,14 +182,7 @@ public class LuceneEventIndex implements EventIndex {
for (final File defunctIndex : defunctIndices) {
try {
if (isLucene4IndexPresent(defunctIndex)) {
logger.info("Encountered Lucene 8 index {} and also the corresponding Lucene 4 index; will only trigger rebuilding of one directory.", defunctIndex);
rebuildCount.incrementAndGet();
continue;
}
logger.info("Determined that Lucene Index Directory {} is defunct. Will destroy and rebuild index", defunctIndex);
final Tuple<Long, Long> timeRange = getTimeRange(defunctIndex, allIndexDirectories);
rebuildIndexExecutor.submit(new MigrateDefunctIndex(defunctIndex, indexManager, directoryManager, timeRange.getKey(), timeRange.getValue(),
eventStore, eventReporter, eventConverter, rebuildCount, totalCount));
@ -208,28 +201,6 @@ public class LuceneEventIndex implements EventIndex {
}
}
/**
* Returns true if the given Index Directory appears to be a later version of the Lucene Index and there also exists a version 4 Lucene
* Index for the same timestamp
* @param indexDirectory the index directory to check
* @return <code>true</code> if there exists a Lucene 4 index directory for the same timestamp, <code>false</code> otherwise
*/
private boolean isLucene4IndexPresent(final File indexDirectory) {
final String indexName = indexDirectory.getName();
if (indexName.contains("lucene-8-")) {
final int prefixEnd = indexName.indexOf("index-");
final String oldIndexName = indexName.substring(prefixEnd);
final File oldIndexFile = new File(indexDirectory.getParentFile(), oldIndexName);
final boolean oldIndexExists = oldIndexFile.exists();
if (oldIndexExists) {
return true;
}
}
return false;
}
private void triggerCacheWarming() {
final Optional<Integer> warmCacheMinutesOption = config.getWarmCacheFrequencyMinutes();
if (warmCacheMinutesOption.isPresent() && warmCacheMinutesOption.get() > 0) {

13
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/index/lucene/MigrateDefunctIndex.java
View File

@ -36,8 +36,8 @@ import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
public class MigrateDefunctIndex implements Runnable {
private static final String TEMP_FILENAME_PREFIX = "temp-lucene-8-";
private static final String MIGRATED_FILENAME_PREFIX = "lucene-8-";
private static final String TEMP_FILENAME_PREFIX = "temp-lucene-9-";
private static final String MIGRATED_FILENAME_PREFIX = "lucene-9-";
private static final Logger logger = LoggerFactory.getLogger(MigrateDefunctIndex.class);
private final File indexDirectory;
@ -71,8 +71,9 @@ public class MigrateDefunctIndex implements Runnable {
@Override
public void run() {
final File tempIndexDir = new File(indexDirectory.getParentFile(), TEMP_FILENAME_PREFIX + indexDirectory.getName());
final File migratedIndexDir = new File(indexDirectory.getParentFile(), MIGRATED_FILENAME_PREFIX + indexDirectory.getName());
final String indexDirSuffix = indexDirectory.getName().substring(MIGRATED_FILENAME_PREFIX.length());
final File tempIndexDir = new File(indexDirectory.getParentFile(), TEMP_FILENAME_PREFIX + indexDirSuffix);
final File migratedIndexDir = new File(indexDirectory.getParentFile(), MIGRATED_FILENAME_PREFIX + indexDirSuffix);
final boolean preconditionsMet = verifyPreconditions(tempIndexDir, migratedIndexDir);
if (!preconditionsMet) {
@ -101,7 +102,7 @@ public class MigrateDefunctIndex implements Runnable {
try {
FileUtils.deleteFile(tempIndexDir, true);
} catch (final Exception e) {
logger.error("Attempted to rebuild index for {} but there already exists a temporary Lucene 8 index at {}. " +
logger.error("Attempted to rebuild index for {} but there already exists a temporary Lucene 9 index at {}. " +
"Attempted to delete existing temp directory but failed. This index will not be rebuilt.", tempIndexDir, e);
return false;
}
@ -112,7 +113,7 @@ public class MigrateDefunctIndex implements Runnable {
try {
FileUtils.deleteFile(migratedIndexDir, true);
} catch (final Exception e) {
logger.error("Attempted to rebuild index for {} but there already exists a Lucene 8 index at {}. " +
logger.error("Attempted to rebuild index for {} but there already exists a Lucene 9 index at {}. " +
"Attempted to delete existing Lucene 8 directory but failed. This index will not be rebuilt.", migratedIndexDir, e);
return false;
}

9
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/StartupAgainstOldLuceneIndexIT.java
View File

@ -35,7 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* With NiFi 1.10.0 (?) we changed from Lucene 4.x to Lucene 8.x
* This test is intended to ensure that we can properly startup even when pointing to a Provenance
* Repository that was created against the old Lucene.
*/
@ -59,11 +58,11 @@ public class StartupAgainstOldLuceneIndexIT {
final File tempDir = new File("target/" + UUID.randomUUID());
copy(existingRepo, tempDir);
final File oldIndexDir = new File(tempDir, "index-1554304717707");
final File oldIndexDir = new File(tempDir, "lucene-4-index-1554304717707");
assertTrue(oldIndexDir.exists());
if (createTempDirectory) {
final File tempIndexDir = new File(tempDir, "temp-lucene-8-index-1554304717707");
final File tempIndexDir = new File(tempDir, "temp-lucene-9-index-1554304717707");
assertTrue(tempIndexDir.mkdirs());
final File dummyFile = new File(tempIndexDir, "_0.fdt");
@ -73,7 +72,7 @@ public class StartupAgainstOldLuceneIndexIT {
}
if (createMigratedDirectory) {
final File migratedDirectory = new File(tempDir, "lucene-8-index-1554304717707");
final File migratedDirectory = new File(tempDir, "lucene-9-index-1554304717707");
assertTrue(migratedDirectory.mkdirs());
final File dummyFile = new File(migratedDirectory, "_0.fdt");
@ -99,7 +98,7 @@ public class StartupAgainstOldLuceneIndexIT {
assertFalse(oldIndexDir.exists());
final File newIndexDir = new File(tempDir, "lucene-8-index-1554304717707");
final File newIndexDir = new File(tempDir, "lucene-9-index-1554304717707");
while (!newIndexDir.exists()) {
Thread.sleep(5L);
}

26
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/index/lucene/TestIndexDirectoryManager.java
View File

@ -42,8 +42,8 @@ public class TestIndexDirectoryManager {
final List<File> directories = IndexDirectoryManager.getDirectories(1000L, 1001L, locations);
assertEquals(2, directories.size());
assertTrue(directories.contains(new File("lucene-8-index-999")));
assertTrue(directories.contains(new File("lucene-8-index-1002")));
assertTrue(directories.contains(new File("lucene-9-index-999")));
assertTrue(directories.contains(new File("lucene-9-index-1002")));
}
@Test
@ -53,10 +53,10 @@ public class TestIndexDirectoryManager {
final File storageDir1 = config.getStorageDirectories().get("1");
final File storageDir2 = config.getStorageDirectories().get("2");
final File index1 = new File(storageDir1, "lucene-8-index-1");
final File index2 = new File(storageDir1, "lucene-8-index-2");
final File index3 = new File(storageDir2, "lucene-8-index-3");
final File index4 = new File(storageDir2, "lucene-8-index-4");
final File index1 = new File(storageDir1, "lucene-9-index-1");
final File index2 = new File(storageDir1, "lucene-9-index-2");
final File index3 = new File(storageDir2, "lucene-9-index-3");
final File index4 = new File(storageDir2, "lucene-9-index-4");
final File[] allIndices = new File[] {index1, index2, index3, index4};
for (final File file : allIndices) {
@ -93,10 +93,10 @@ public class TestIndexDirectoryManager {
final File storageDir1 = config.getStorageDirectories().get("1");
final File storageDir2 = config.getStorageDirectories().get("2");
final File index1 = new File(storageDir1, "lucene-8-index-1");
final File index2 = new File(storageDir1, "lucene-8-index-2");
final File index3 = new File(storageDir2, "lucene-8-index-3");
final File index4 = new File(storageDir2, "lucene-8-index-4");
final File index1 = new File(storageDir1, "lucene-9-index-1");
final File index2 = new File(storageDir1, "lucene-9-index-2");
final File index3 = new File(storageDir2, "lucene-9-index-3");
final File index4 = new File(storageDir2, "lucene-9-index-4");
final File[] allIndices = new File[] {index1, index2, index3, index4};
for (final File file : allIndices) {
@ -138,8 +138,8 @@ public class TestIndexDirectoryManager {
final File storageDir = config.getStorageDirectories().get("1");
final File index1 = new File(storageDir, "lucene-8-index-1");
final File index2 = new File(storageDir, "lucene-8-index-2");
final File index1 = new File(storageDir, "lucene-9-index-1");
final File index2 = new File(storageDir, "lucene-9-index-2");
final File[] allIndices = new File[] {index1, index2};
for (final File file : allIndices) {
@ -171,7 +171,7 @@ public class TestIndexDirectoryManager {
}
private IndexLocation createLocation(final long timestamp, final String partitionName) {
return new IndexLocation(new File("lucene-8-index-" + timestamp), timestamp, partitionName);
return new IndexLocation(new File("lucene-9-index-" + timestamp), timestamp, partitionName);
}
private RepositoryConfiguration createConfig(final int partitions) {

26
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/index/lucene/TestLuceneEventIndex.java
View File

@ -83,22 +83,22 @@ public class TestLuceneEventIndex {
final long now = System.currentTimeMillis();
final List<File> indexFiles = new ArrayList<>();
indexFiles.add(new File("index-1000"));
indexFiles.add(new File("lucene-8-index-3000"));
indexFiles.add(new File("index-4000"));
indexFiles.add(new File("index-5000"));
indexFiles.add(new File("lucene-8-index-6000"));
indexFiles.add(new File("index-7000"));
indexFiles.add(new File("lucene-4-index-1000"));
indexFiles.add(new File("lucene-9-index-3000"));
indexFiles.add(new File("lucene-4-index-4000"));
indexFiles.add(new File("lucene-4-index-5000"));
indexFiles.add(new File("lucene-9-index-6000"));
indexFiles.add(new File("lucene-4-index-7000"));
assertEquals(new Tuple<>(1000L, 3000L), LuceneEventIndex.getTimeRange(new File("index-1000"), indexFiles));
assertEquals(new Tuple<>(1000L, 3000L), LuceneEventIndex.getTimeRange(new File("lucene-4-index-1000"), indexFiles));
assertEquals(new Tuple<>(3000L, 4000L), LuceneEventIndex.getTimeRange(new File("lucene-8-index-3000"), indexFiles));
assertEquals(new Tuple<>(4000L, 5000L), LuceneEventIndex.getTimeRange(new File("index-4000"), indexFiles));
assertEquals(new Tuple<>(5000L, 6000L), LuceneEventIndex.getTimeRange(new File("index-5000"), indexFiles));
assertEquals(new Tuple<>(6000L, 7000L), LuceneEventIndex.getTimeRange(new File("lucene-8-index-6000"), indexFiles));
assertEquals(new Tuple<>(3000L, 4000L), LuceneEventIndex.getTimeRange(new File("lucene-9-index-3000"), indexFiles));
assertEquals(new Tuple<>(4000L, 5000L), LuceneEventIndex.getTimeRange(new File("lucene-4-index-4000"), indexFiles));
assertEquals(new Tuple<>(5000L, 6000L), LuceneEventIndex.getTimeRange(new File("lucene-4-index-5000"), indexFiles));
assertEquals(new Tuple<>(6000L, 7000L), LuceneEventIndex.getTimeRange(new File("lucene-9-index-6000"), indexFiles));
assertEquals(7000L, LuceneEventIndex.getTimeRange(new File("index-7000"), indexFiles).getKey().longValue());
assertTrue(LuceneEventIndex.getTimeRange(new File("index-7000"), indexFiles).getValue() >= now);
assertEquals(7000L, LuceneEventIndex.getTimeRange(new File("lucene-4-index-7000"), indexFiles).getKey().longValue());
assertTrue(LuceneEventIndex.getTimeRange(new File("lucene-4-index-7000"), indexFiles).getValue() >= now);
}

0
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/index-1554304717707/_0.fdt → nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_0.fdt
View File

0
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/index-1554304717707/_0.fdx → nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_0.fdx
View File

0
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/index-1554304717707/_1.fdt → nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_1.fdt
View File

0
nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/index-1554304717707/_1.fdx → nifi-extension-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_1.fdx
View File

10
nifi-extension-bundles/nifi-provenance-repository-bundle/pom.xml
View File

@ -46,16 +46,6 @@
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>