Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-2065: When a provenance query matches the max number of results requested, stop querying lucene for improved performance 

This closes #558
This commit is contained in:
Mark Payne 2016-06-21 17:34:15 -04:00 committed by Matt Burgess
parent 36ab8474dd
commit 0e085bdddd
6 changed files with 36 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ProvenanceQueryEndpointMerger.java
View File

@ -176,8 +176,14 @@ public class ProvenanceQueryEndpointMerger implements EndpointResponseMerger {
results.setErrors(errors);
}
results.setTotalCount(totalRecords);
results.setTotal(FormatUtils.formatCount(totalRecords));
if (clientDto.getRequest().getMaxResults() != null && totalRecords >= clientDto.getRequest().getMaxResults()) {
results.setTotalCount(clientDto.getRequest().getMaxResults().longValue());
results.setTotal(FormatUtils.formatCount(clientDto.getRequest().getMaxResults().longValue()) + "+");
} else {
results.setTotal(FormatUtils.formatCount(totalRecords));
results.setTotalCount(totalRecords);
}
results.setProvenanceEvents(selectedResults);
results.setOldestEvent(oldestEventDate);
results.setGenerated(new Date());

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
View File

@ -964,8 +964,14 @@ public class ControllerFacade implements Authorizable {
events.add(createProvenanceEventDto(record));
}
resultsDto.setProvenanceEvents(events);
resultsDto.setTotalCount(queryResult.getTotalHitCount());
resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) {
resultsDto.setTotalCount(requestDto.getMaxResults().longValue());
resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+");
} else {
resultsDto.setTotalCount(queryResult.getTotalHitCount());
resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount()));
}
// include any errors
if (queryResult.getError() != null) {

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js
View File

@ -988,7 +988,7 @@ nf.ProvenanceTable = (function () {
// update the filter message based on the request
if (isBlankQuery(provenanceRequest)) {
var message = 'Showing the most recent ';
if (provenanceResults.totalCount > config.maxResults) {
if (provenanceResults.totalCount >= config.maxResults) {
message += (nf.Common.formatInteger(config.maxResults) + ' of ' + provenanceResults.total + ' events, please refine the search.');
} else {
message += ('events.');
@ -997,7 +997,7 @@ nf.ProvenanceTable = (function () {
$('#clear-provenance-search').hide();
} else {
var message = 'Showing ';
if (provenanceResults.totalCount > config.maxResults) {
if (provenanceResults.totalCount >= config.maxResults) {
message += (nf.Common.formatInteger(config.maxResults) + ' of ' + provenanceResults.total + ' events that match the specified query, please refine the search.');
} else {
message += ('the events that match the specified query.');

4
nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java
View File

@ -2330,10 +2330,6 @@ public class PersistentProvenanceRepository implements ProvenanceEventRepository
final IndexSearch search = new IndexSearch(PersistentProvenanceRepository.this, indexDir, indexManager, maxAttributeChars);
final StandardQueryResult queryResult = search.search(query, retrievalCount, firstEventTimestamp);
submission.getResult().update(queryResult.getMatchingEvents(), queryResult.getTotalHitCount());
if (queryResult.isFinished()) {
logger.info("Successfully executed Query[{}] against Index {}; Search took {} milliseconds; Total Hits = {}",
query, indexDir, queryResult.getQueryTime(), queryResult.getTotalHitCount());
}
} catch (final Throwable t) {
logger.error("Failed to query Provenance Repository Index {} due to {}", indexDir, t.toString());
if (logger.isDebugEnabled()) {

2
nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java
View File

@ -127,7 +127,7 @@ class DocsReader {
maxAttributeChars)) {
Iterator<Document> docIter = byStorageNameDocGroups.get(storageFileName).iterator();
while (docIter.hasNext() && retrievalCount.incrementAndGet() < maxResults){
while (docIter.hasNext() && retrievalCount.getAndIncrement() < maxResults) {
ProvenanceEventRecord eRec = this.getRecord(docIter.next(), reader);
if (eRec != null) {
matchingRecords.add(eRec);

17
nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java
View File

@ -49,6 +49,18 @@ public class IndexSearch {
}
public StandardQueryResult search(final org.apache.nifi.provenance.search.Query provenanceQuery, final AtomicInteger retrievedCount, final long firstEventTimestamp) throws IOException {
if (retrievedCount.get() >= provenanceQuery.getMaxResults()) {
final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1);
sqr.update(Collections.<ProvenanceEventRecord> emptyList(), 0L);
logger.info("Skipping search of Provenance Index {} for {} because the max number of results ({}) has already been retrieved",
indexDirectory, provenanceQuery, provenanceQuery.getMaxResults());
return sqr;
}
final long startNanos = System.nanoTime();
if (!indexDirectory.exists() && !indexDirectory.mkdirs()) {
throw new IOException("Unable to create Indexing Directory " + indexDirectory);
}
@ -97,6 +109,11 @@ public class IndexSearch {
logger.debug("Reading {} records took {} millis for {}", matchingRecords.size(), TimeUnit.NANOSECONDS.toMillis(readRecordsNanos), this);
sqr.update(matchingRecords, topDocs.totalHits);
final long queryNanos = System.nanoTime() - startNanos;
logger.info("Successfully executed {} against Index {}; Search took {} milliseconds; Total Hits = {}",
provenanceQuery, indexDirectory, TimeUnit.NANOSECONDS.toMillis(queryNanos), topDocs.totalHits);
return sqr;
} catch (final FileNotFoundException e) {
// nothing has been indexed yet, or the data has already aged off