Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-432: Reduced amount of time needed for rollovers in unit tests in order to speed up tests

This commit is contained in:
Mark Payne 2015-03-18 10:10:21 -04:00
parent 71989128b9
commit b560a88cbd
2 changed files with 26 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
View File

@ -130,6 +130,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
private final RepositoryConfiguration configuration; private final RepositoryConfiguration configuration;
private final IndexConfiguration indexConfig; private final IndexConfiguration indexConfig;
private final boolean alwaysSync; private final boolean alwaysSync;
private final int rolloverCheckMillis;
private final ScheduledExecutorService scheduledExecService; private final ScheduledExecutorService scheduledExecService;
private final ExecutorService rolloverExecutor; private final ExecutorService rolloverExecutor;
@ -156,10 +157,10 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
private EventReporter eventReporter; private EventReporter eventReporter;
public PersistentProvenanceRepository() throws IOException { public PersistentProvenanceRepository() throws IOException {
this(createRepositoryConfiguration()); this(createRepositoryConfiguration(), 10000);
} }
public PersistentProvenanceRepository(final RepositoryConfiguration configuration) throws IOException { public PersistentProvenanceRepository(final RepositoryConfiguration configuration, final int rolloverCheckMillis) throws IOException {
if (configuration.getStorageDirectories().isEmpty()) { if (configuration.getStorageDirectories().isEmpty()) {
throw new IllegalArgumentException("Must specify at least one storage directory"); throw new IllegalArgumentException("Must specify at least one storage directory");
} }
@ -181,6 +182,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
this.maxPartitionBytes = configuration.getMaxEventFileCapacity(); this.maxPartitionBytes = configuration.getMaxEventFileCapacity();
this.indexConfig = new IndexConfiguration(configuration); this.indexConfig = new IndexConfiguration(configuration);
this.alwaysSync = configuration.isAlwaysSync(); this.alwaysSync = configuration.isAlwaysSync();
this.rolloverCheckMillis = rolloverCheckMillis;
final List<SearchableField> fields = configuration.getSearchableFields(); final List<SearchableField> fields = configuration.getSearchableFields();
if (fields != null && !fields.isEmpty()) { if (fields != null && !fields.isEmpty()) {
@ -244,7 +246,7 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
} }
} }
} }
}, 10L, 10L, TimeUnit.SECONDS); }, rolloverCheckMillis, rolloverCheckMillis, TimeUnit.MILLISECONDS);
scheduledExecService.scheduleWithFixedDelay(new RemoveExpiredQueryResults(), 30L, 3L, TimeUnit.SECONDS); scheduledExecService.scheduleWithFixedDelay(new RemoveExpiredQueryResults(), 30L, 3L, TimeUnit.SECONDS);
scheduledExecService.scheduleWithFixedDelay(new Runnable() { scheduledExecService.scheduleWithFixedDelay(new Runnable() {

40
nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
View File

@ -170,7 +170,7 @@ public class TestPersistentProvenanceRepository {
config.setCompressOnRollover(false); config.setCompressOnRollover(false);
config.setJournalCount(10); config.setJournalCount(10);
config.setQueryThreadPoolSize(10); config.setQueryThreadPoolSize(10);
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final Map<String, String> attributes = new HashMap<>(); final Map<String, String> attributes = new HashMap<>();
@ -218,7 +218,7 @@ public class TestPersistentProvenanceRepository {
System.out.println("Closing and re-initializing"); System.out.println("Closing and re-initializing");
repo.close(); repo.close();
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
System.out.println("Re-initialized"); System.out.println("Re-initialized");
@ -238,7 +238,7 @@ public class TestPersistentProvenanceRepository {
final RepositoryConfiguration config = createConfiguration(); final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileCapacity(1L); config.setMaxEventFileCapacity(1L);
config.setMaxEventFileLife(1, TimeUnit.SECONDS); config.setMaxEventFileLife(1, TimeUnit.SECONDS);
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final Map<String, String> attributes = new HashMap<>(); final Map<String, String> attributes = new HashMap<>();
@ -262,7 +262,7 @@ public class TestPersistentProvenanceRepository {
repo.close(); repo.close();
Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.) Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final List<ProvenanceEventRecord> recoveredRecords = repo.getEvents(0L, 12); final List<ProvenanceEventRecord> recoveredRecords = repo.getEvents(0L, 12);
@ -285,7 +285,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setMaxEventFileLife(2, TimeUnit.SECONDS); config.setMaxEventFileLife(2, TimeUnit.SECONDS);
config.setSearchableFields(searchableFields); config.setSearchableFields(searchableFields);
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final Map<String, String> attributes = new HashMap<>(); final Map<String, String> attributes = new HashMap<>();
@ -339,7 +339,7 @@ public class TestPersistentProvenanceRepository {
final RepositoryConfiguration config = createConfiguration(); final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -383,7 +383,7 @@ public class TestPersistentProvenanceRepository {
final RepositoryConfiguration config = createConfiguration(); final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
config.setCompressOnRollover(true); config.setCompressOnRollover(true);
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -421,7 +421,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "10000000-0000-0000-0000-000000000000"; final String uuid = "10000000-0000-0000-0000-000000000000";
@ -480,7 +480,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -543,7 +543,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -625,7 +625,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -681,7 +681,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000001"; final String uuid = "00000000-0000-0000-0000-000000000001";
@ -736,7 +736,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000001"; final String uuid = "00000000-0000-0000-0000-000000000001";
@ -795,7 +795,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String childId = "00000000-0000-0000-0000-000000000000"; final String childId = "00000000-0000-0000-0000-000000000000";
@ -845,7 +845,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String childId = "00000000-0000-0000-0000-000000000000"; final String childId = "00000000-0000-0000-0000-000000000000";
@ -892,7 +892,7 @@ public class TestPersistentProvenanceRepository {
public void testCorrectProvenanceEventIdOnRestore() throws IOException { public void testCorrectProvenanceEventIdOnRestore() throws IOException {
final RepositoryConfiguration config = createConfiguration(); final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileLife(1, TimeUnit.SECONDS); config.setMaxEventFileLife(1, TimeUnit.SECONDS);
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -918,7 +918,7 @@ public class TestPersistentProvenanceRepository {
repo.close(); repo.close();
final PersistentProvenanceRepository secondRepo = new PersistentProvenanceRepository(config); final PersistentProvenanceRepository secondRepo = new PersistentProvenanceRepository(config, 75);
secondRepo.initialize(getEventReporter()); secondRepo.initialize(getEventReporter());
final ProvenanceEventRecord event11 = builder.build(); final ProvenanceEventRecord event11 = builder.build();
@ -938,7 +938,7 @@ public class TestPersistentProvenanceRepository {
config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileCapacity(1024L * 1024L);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -1002,7 +1002,7 @@ public class TestPersistentProvenanceRepository {
final RepositoryConfiguration config = createConfiguration(); final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS);
config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields()));
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final String uuid = "00000000-0000-0000-0000-000000000000"; final String uuid = "00000000-0000-0000-0000-000000000000";
@ -1070,7 +1070,7 @@ public class TestPersistentProvenanceRepository {
public void testMergeJournals() throws IOException, InterruptedException { public void testMergeJournals() throws IOException, InterruptedException {
final RepositoryConfiguration config = createConfiguration(); final RepositoryConfiguration config = createConfiguration();
config.setMaxEventFileLife(3, TimeUnit.SECONDS); config.setMaxEventFileLife(3, TimeUnit.SECONDS);
repo = new PersistentProvenanceRepository(config); repo = new PersistentProvenanceRepository(config, 75);
repo.initialize(getEventReporter()); repo.initialize(getEventReporter());
final Map<String, String> attributes = new HashMap<>(); final Map<String, String> attributes = new HashMap<>();