diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/provenance/ProvenanceRequestDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/provenance/ProvenanceRequestDTO.java index 0cdcb8fdd2..ae54a15ec7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/provenance/ProvenanceRequestDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/provenance/ProvenanceRequestDTO.java @@ -38,6 +38,9 @@ public class ProvenanceRequestDTO { private String maximumFileSize; private Integer maxResults; + private Boolean summarize; + private Boolean incrementalResults; + /** * @return the search terms to use for this search */ @@ -139,4 +142,34 @@ public class ProvenanceRequestDTO { public void setClusterNodeId(String clusterNodeId) { this.clusterNodeId = clusterNodeId; } + + /** + * @return whether or not incremental results are returned. If false, provenance events + * are only returned once the query completes. This property is true by default. + */ + @ApiModelProperty( + value = "Whether or not incremental results are returned. If false, provenance events" + + " are only returned once the query completes. This property is true by default." + ) + public Boolean getIncrementalResults() { + return incrementalResults; + } + + public void setIncrementalResults(Boolean incrementalResults) { + this.incrementalResults = incrementalResults; + } + + /** + * @return whether or not to summarize provenance events returned. This property is false by default. + */ + @ApiModelProperty( + value = "Whether or not to summarize provenance events returned. This property is false by default." + ) + public Boolean getSummarize() { + return summarize; + } + + public void setSummarize(Boolean summarize) { + this.summarize = summarize; + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 50c36c5fdd..1a72127e21 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -206,9 +206,11 @@ public interface NiFiServiceFacade { * Retrieves provenance. * * @param queryId identifier + * @param summarize whether to summarize the event dtos + * @param incrementalResults whether to return any events if the search has not finished * @return result */ - ProvenanceDTO getProvenance(String queryId); + ProvenanceDTO getProvenance(String queryId, Boolean summarize, Boolean incrementalResults); /** * Deletes provenance. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index c8aad318a1..7554dfb54a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -2148,8 +2148,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { } @Override - public ProvenanceDTO getProvenance(final String queryId) { - return controllerFacade.getProvenanceQuery(queryId); + public ProvenanceDTO getProvenance(final String queryId, final Boolean summarize, final Boolean incrementalResults) { + return controllerFacade.getProvenanceQuery(queryId, summarize, incrementalResults); } @Override diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceEventResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceEventResource.java index 6f424f4642..8b1ebf50bb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceEventResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceEventResource.java @@ -22,7 +22,9 @@ import com.wordnik.swagger.annotations.ApiParam; import com.wordnik.swagger.annotations.ApiResponse; import com.wordnik.swagger.annotations.ApiResponses; import com.wordnik.swagger.annotations.Authorization; +import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.coordination.http.replication.RequestReplicator; +import org.apache.nifi.cluster.protocol.NodeIdentifier; import org.apache.nifi.controller.repository.claim.ContentDirection; import org.apache.nifi.stream.io.StreamUtils; import org.apache.nifi.web.DownloadableContent; @@ -286,6 +288,13 @@ public class ProvenanceEventResource extends ApplicationResource { final ProvenanceEventDTO event = serviceFacade.getProvenanceEvent(id.getLong()); event.setClusterNodeId(clusterNodeId); + // populate the cluster node address + final ClusterCoordinator coordinator = getClusterCoordinator(); + if (coordinator != null) { + final NodeIdentifier nodeId = coordinator.getNodeIdentifier(clusterNodeId); + event.setClusterNodeAddress(nodeId.getApiAddress() + ":" + nodeId.getApiPort()); + } + // create a response entity final ProvenanceEventEntity entity = new ProvenanceEventEntity(); entity.setProvenanceEvent(event); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java index 9044bbe8b9..30c1896e67 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProvenanceResource.java @@ -46,6 +46,7 @@ import org.apache.nifi.web.api.entity.ProvenanceOptionsEntity; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.HttpMethod; import javax.ws.rs.POST; @@ -294,6 +295,17 @@ public class ProvenanceResource extends ApplicationResource { required = false ) @QueryParam("clusterNodeId") final String clusterNodeId, + @ApiParam( + value = "Whether or not incremental results are returned. If false, provenance events" + + " are only returned once the query completes. This property is true by default.", + required = false + ) + @QueryParam("summarize") @DefaultValue(value = "false") final Boolean summarize, + @ApiParam( + value = "Whether or not to summarize provenance events returned. This property is false by default.", + required = false + ) + @QueryParam("incrementalResults") @DefaultValue(value = "true") final Boolean incrementalResults, @ApiParam( value = "The id of the provenance query.", required = true @@ -314,7 +326,7 @@ public class ProvenanceResource extends ApplicationResource { } // get the provenance - final ProvenanceDTO dto = serviceFacade.getProvenance(id); + final ProvenanceDTO dto = serviceFacade.getProvenance(id, summarize, incrementalResults); dto.getRequest().setClusterNodeId(clusterNodeId); populateRemainingProvenanceContent(dto); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java index 93d86c9a95..2ad5303a84 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java @@ -972,7 +972,7 @@ public class ControllerFacade implements Authorizable { final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser()); // return the query with the results populated at this point - return getProvenanceQuery(querySubmission.getQueryIdentifier()); + return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults()); } /** @@ -981,7 +981,7 @@ public class ControllerFacade implements Authorizable { * @param provenanceId id * @return the results of a provenance query */ - public ProvenanceDTO getProvenanceQuery(String provenanceId) { + public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) { try { // get the query to the provenance repository final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository(); @@ -1027,11 +1027,14 @@ public class ControllerFacade implements Authorizable { provenanceDto.setPercentCompleted(queryResult.getPercentComplete()); // convert each event - final List events = new ArrayList<>(); - for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) { - events.add(createProvenanceEventDto(record)); + final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults); + if (includeResults || queryResult.isFinished()) { + final List events = new ArrayList<>(); + for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) { + events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize))); + } + resultsDto.setProvenanceEvents(events); } - resultsDto.setProvenanceEvents(events); if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) { resultsDto.setTotalCount(requestDto.getMaxResults().longValue()); @@ -1227,7 +1230,7 @@ public class ControllerFacade implements Authorizable { final ProvenanceEventRecord event = flowController.replayFlowFile(originalEvent, user); // convert the event record - return createProvenanceEventDto(event); + return createProvenanceEventDto(event, false); } catch (final IOException ioe) { throw new NiFiCoreException("An error occurred while getting the specified event.", ioe); } @@ -1313,7 +1316,7 @@ public class ControllerFacade implements Authorizable { dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser(), attributes); // convert the event - return createProvenanceEventDto(event); + return createProvenanceEventDto(event, false); } catch (final IOException ioe) { throw new NiFiCoreException("An error occurred while getting the specified event.", ioe); } @@ -1325,117 +1328,121 @@ public class ControllerFacade implements Authorizable { * @param event event * @return event */ - private ProvenanceEventDTO createProvenanceEventDto(final ProvenanceEventRecord event) { - // convert the attributes - final Comparator attributeComparator = new Comparator() { - @Override - public int compare(AttributeDTO a1, AttributeDTO a2) { - return Collator.getInstance(Locale.US).compare(a1.getName(), a2.getName()); - } - }; - - final SortedSet attributes = new TreeSet<>(attributeComparator); - - final Map updatedAttrs = event.getUpdatedAttributes(); - final Map previousAttrs = event.getPreviousAttributes(); - - // add previous attributes that haven't been modified. - for (final Map.Entry entry : previousAttrs.entrySet()) { - // don't add any attributes that have been updated; we will do that next - if (updatedAttrs.containsKey(entry.getKey())) { - continue; - } - - final AttributeDTO attribute = new AttributeDTO(); - attribute.setName(entry.getKey()); - attribute.setValue(entry.getValue()); - attribute.setPreviousValue(entry.getValue()); - attributes.add(attribute); - } - - // Add all of the update attributes - for (final Map.Entry entry : updatedAttrs.entrySet()) { - final AttributeDTO attribute = new AttributeDTO(); - attribute.setName(entry.getKey()); - attribute.setValue(entry.getValue()); - attribute.setPreviousValue(previousAttrs.get(entry.getKey())); - attributes.add(attribute); - } - - // build the event dto + private ProvenanceEventDTO createProvenanceEventDto(final ProvenanceEventRecord event, final boolean summarize) { final ProvenanceEventDTO dto = new ProvenanceEventDTO(); dto.setId(String.valueOf(event.getEventId())); - dto.setAlternateIdentifierUri(event.getAlternateIdentifierUri()); - dto.setAttributes(attributes); - dto.setTransitUri(event.getTransitUri()); dto.setEventId(event.getEventId()); dto.setEventTime(new Date(event.getEventTime())); dto.setEventType(event.getEventType().name()); + dto.setFlowFileUuid(event.getFlowFileUuid()); dto.setFileSize(FormatUtils.formatDataSize(event.getFileSize())); dto.setFileSizeBytes(event.getFileSize()); dto.setComponentId(event.getComponentId()); dto.setComponentType(event.getComponentType()); - dto.setSourceSystemFlowFileId(event.getSourceSystemFlowFileIdentifier()); - dto.setFlowFileUuid(event.getFlowFileUuid()); - dto.setRelationship(event.getRelationship()); - dto.setDetails(event.getDetails()); - - final ContentAvailability contentAvailability = flowController.getContentAvailability(event); - - // content - dto.setContentEqual(contentAvailability.isContentSame()); - dto.setInputContentAvailable(contentAvailability.isInputAvailable()); - dto.setInputContentClaimSection(event.getPreviousContentClaimSection()); - dto.setInputContentClaimContainer(event.getPreviousContentClaimContainer()); - dto.setInputContentClaimIdentifier(event.getPreviousContentClaimIdentifier()); - dto.setInputContentClaimOffset(event.getPreviousContentClaimOffset()); - dto.setInputContentClaimFileSizeBytes(event.getPreviousFileSize()); - dto.setOutputContentAvailable(contentAvailability.isOutputAvailable()); - dto.setOutputContentClaimSection(event.getContentClaimSection()); - dto.setOutputContentClaimContainer(event.getContentClaimContainer()); - dto.setOutputContentClaimIdentifier(event.getContentClaimIdentifier()); - dto.setOutputContentClaimOffset(event.getContentClaimOffset()); - dto.setOutputContentClaimFileSize(FormatUtils.formatDataSize(event.getFileSize())); - dto.setOutputContentClaimFileSizeBytes(event.getFileSize()); - - // format the previous file sizes if possible - if (event.getPreviousFileSize() != null) { - dto.setInputContentClaimFileSize(FormatUtils.formatDataSize(event.getPreviousFileSize())); - } - - // determine if authorized for event replay - final AuthorizationResult replayAuthorized = checkAuthorizationForReplay(event); - - // replay - dto.setReplayAvailable(contentAvailability.isReplayable() && Result.Approved.equals(replayAuthorized.getResult())); - dto.setReplayExplanation(contentAvailability.isReplayable() - && !Result.Approved.equals(replayAuthorized.getResult()) ? replayAuthorized.getExplanation() : contentAvailability.getReasonNotReplayable()); - dto.setSourceConnectionIdentifier(event.getSourceQueueIdentifier()); // sets the component details if it can find the component still in the flow setComponentDetails(dto); - // event duration - if (event.getEventDuration() >= 0) { - dto.setEventDuration(event.getEventDuration()); + // only include all details if not summarizing + if (!summarize) { + // convert the attributes + final Comparator attributeComparator = new Comparator() { + @Override + public int compare(AttributeDTO a1, AttributeDTO a2) { + return Collator.getInstance(Locale.US).compare(a1.getName(), a2.getName()); + } + }; + + final SortedSet attributes = new TreeSet<>(attributeComparator); + + final Map updatedAttrs = event.getUpdatedAttributes(); + final Map previousAttrs = event.getPreviousAttributes(); + + // add previous attributes that haven't been modified. + for (final Map.Entry entry : previousAttrs.entrySet()) { + // don't add any attributes that have been updated; we will do that next + if (updatedAttrs.containsKey(entry.getKey())) { + continue; + } + + final AttributeDTO attribute = new AttributeDTO(); + attribute.setName(entry.getKey()); + attribute.setValue(entry.getValue()); + attribute.setPreviousValue(entry.getValue()); + attributes.add(attribute); + } + + // Add all of the update attributes + for (final Map.Entry entry : updatedAttrs.entrySet()) { + final AttributeDTO attribute = new AttributeDTO(); + attribute.setName(entry.getKey()); + attribute.setValue(entry.getValue()); + attribute.setPreviousValue(previousAttrs.get(entry.getKey())); + attributes.add(attribute); + } + + // additional event details + dto.setAlternateIdentifierUri(event.getAlternateIdentifierUri()); + dto.setAttributes(attributes); + dto.setTransitUri(event.getTransitUri()); + dto.setSourceSystemFlowFileId(event.getSourceSystemFlowFileIdentifier()); + dto.setRelationship(event.getRelationship()); + dto.setDetails(event.getDetails()); + + final ContentAvailability contentAvailability = flowController.getContentAvailability(event); + + // content + dto.setContentEqual(contentAvailability.isContentSame()); + dto.setInputContentAvailable(contentAvailability.isInputAvailable()); + dto.setInputContentClaimSection(event.getPreviousContentClaimSection()); + dto.setInputContentClaimContainer(event.getPreviousContentClaimContainer()); + dto.setInputContentClaimIdentifier(event.getPreviousContentClaimIdentifier()); + dto.setInputContentClaimOffset(event.getPreviousContentClaimOffset()); + dto.setInputContentClaimFileSizeBytes(event.getPreviousFileSize()); + dto.setOutputContentAvailable(contentAvailability.isOutputAvailable()); + dto.setOutputContentClaimSection(event.getContentClaimSection()); + dto.setOutputContentClaimContainer(event.getContentClaimContainer()); + dto.setOutputContentClaimIdentifier(event.getContentClaimIdentifier()); + dto.setOutputContentClaimOffset(event.getContentClaimOffset()); + dto.setOutputContentClaimFileSize(FormatUtils.formatDataSize(event.getFileSize())); + dto.setOutputContentClaimFileSizeBytes(event.getFileSize()); + + // format the previous file sizes if possible + if (event.getPreviousFileSize() != null) { + dto.setInputContentClaimFileSize(FormatUtils.formatDataSize(event.getPreviousFileSize())); + } + + // determine if authorized for event replay + final AuthorizationResult replayAuthorized = checkAuthorizationForReplay(event); + + // replay + dto.setReplayAvailable(contentAvailability.isReplayable() && Result.Approved.equals(replayAuthorized.getResult())); + dto.setReplayExplanation(contentAvailability.isReplayable() + && !Result.Approved.equals(replayAuthorized.getResult()) ? replayAuthorized.getExplanation() : contentAvailability.getReasonNotReplayable()); + dto.setSourceConnectionIdentifier(event.getSourceQueueIdentifier()); + + // event duration + if (event.getEventDuration() >= 0) { + dto.setEventDuration(event.getEventDuration()); + } + + // lineage duration + if (event.getLineageStartDate() > 0) { + final long lineageDuration = event.getEventTime() - event.getLineageStartDate(); + dto.setLineageDuration(lineageDuration); + } + + // parent uuids + final List parentUuids = new ArrayList<>(event.getParentUuids()); + Collections.sort(parentUuids, Collator.getInstance(Locale.US)); + dto.setParentUuids(parentUuids); + + // child uuids + final List childUuids = new ArrayList<>(event.getChildUuids()); + Collections.sort(childUuids, Collator.getInstance(Locale.US)); + dto.setChildUuids(childUuids); } - // lineage duration - if (event.getLineageStartDate() > 0) { - final long lineageDuration = event.getEventTime() - event.getLineageStartDate(); - dto.setLineageDuration(lineageDuration); - } - - // parent uuids - final List parentUuids = new ArrayList<>(event.getParentUuids()); - Collections.sort(parentUuids, Collator.getInstance(Locale.US)); - dto.setParentUuids(parentUuids); - - // child uuids - final List childUuids = new ArrayList<>(event.getChildUuids()); - Collections.sort(childUuids, Collator.getInstance(Locale.US)); - dto.setChildUuids(childUuids); - return dto; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js index 4e15cd21fa..d20737cebd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js @@ -26,8 +26,7 @@ nf.ng.ProvenanceLineage = function () { var config = { sliderTickCount: 75, urls: { - lineage: '../nifi-api/provenance/lineage', - events: '../nifi-api/provenance-events/' + lineage: '../nifi-api/provenance/lineage' } }; @@ -90,41 +89,6 @@ nf.ng.ProvenanceLineage = function () { }); }; - /** - * Shows the details for the specified event. - * - * @param {string} eventId - * @param {string} clusterNodeId The id of the node in the cluster where this event/flowfile originated - */ - var showEventDetails = function (eventId, clusterNodeId, provenanceTableCtrl) { - getEventDetails(eventId, clusterNodeId).done(function (response) { - provenanceTableCtrl.showEventDetails(response.provenanceEvent); - }); - }; - - /** - * Gets the details for the specified event. - * - * @param {string} eventId - * @param {string} clusterNodeId The id of the node in the cluster where this event/flowfile originated - */ - var getEventDetails = function (eventId, clusterNodeId) { - var url; - if (nf.Common.isDefinedAndNotNull(clusterNodeId)) { - url = config.urls.events + encodeURIComponent(eventId) + '?' + $.param({ - clusterNodeId: clusterNodeId - }); - } else { - url = config.urls.events + encodeURIComponent(eventId); - } - - return $.ajax({ - type: 'GET', - url: url, - dataType: 'json' - }).fail(nf.Common.handleAjaxError); - }; - /** * Submits the specified lineage request. * @@ -792,7 +756,7 @@ nf.ng.ProvenanceLineage = function () { 'class': 'lineage-view-event', 'text': 'View details', 'click': function () { - showEventDetails(d.id, clusterNodeId, provenanceTableCtrl); + provenanceTableCtrl.showEventDetails(d.id, clusterNodeId); } }]; @@ -852,17 +816,17 @@ nf.ng.ProvenanceLineage = function () { }; // polls for the event lineage - var pollLineage = function (nextDelay) { + var pollLineage = function () { getLineage(lineage).done(function (response) { lineage = response.lineage; - // process the lineage, if its not done computing wait delay seconds before checking again - processLineage(nextDelay); + // process the lineage + processLineage(); }).fail(closeDialog); }; // processes the event lineage - var processLineage = function (delay) { + var processLineage = function () { // if the request was cancelled just ignore the current response if (cancelled === true) { closeDialog(); @@ -907,13 +871,9 @@ nf.ng.ProvenanceLineage = function () { // clear the timer since we've been invoked lineageTimer = null; - // calculate the next delay (back off) - var backoff = delay * 2; - var nextDelay = backoff > provenanceTableCtrl.MAX_DELAY ? provenanceTableCtrl.MAX_DELAY : backoff; - // for the lineage - pollLineage(nextDelay); - }, delay * 1000); + pollLineage(); + }, 2000); } }; @@ -934,7 +894,7 @@ nf.ng.ProvenanceLineage = function () { // collapses the lineage for the specified event in the specified direction var collapseLineage = function (eventId, provenanceTableCtrl) { // get the event in question and collapse in the appropriate direction - getEventDetails(eventId, clusterNodeId).done(function (response) { + provenanceTableCtrl.getEventDetails(eventId, clusterNodeId).done(function (response) { var provenanceEvent = response.provenanceEvent; var eventUuid = provenanceEvent.flowFileUuid; var eventUuids = d3.set(provenanceEvent.childUuids); @@ -1371,18 +1331,17 @@ nf.ng.ProvenanceLineage = function () { $('#lineage-query-dialog').modal('hide'); }; - // polls the server for the status of the lineage, if the lineage is not - // done wait nextDelay seconds before trying again - var pollLineage = function (nextDelay, provenanceTableCtrl) { + // polls the server for the status of the lineage + var pollLineage = function (provenanceTableCtrl) { getLineage(lineage).done(function (response) { lineage = response.lineage; - // process the lineage, if its not done computing wait delay seconds before checking again - processLineage(nextDelay, provenanceTableCtrl); + // process the lineage + processLineage(provenanceTableCtrl); }).fail(closeDialog); }; - var processLineage = function (delay, provenanceTableCtrl) { + var processLineage = function (provenanceTableCtrl) { // if the request was cancelled just ignore the current response if (cancelled === true) { closeDialog(); @@ -1417,13 +1376,9 @@ nf.ng.ProvenanceLineage = function () { // clear the timer since we've been invoked lineageTimer = null; - // calculate the next delay (back off) - var backoff = delay * 2; - var nextDelay = backoff > provenanceTableCtrl.MAX_DELAY ? provenanceTableCtrl.MAX_DELAY : backoff; - // poll lineage - pollLineage(nextDelay, provenanceTableCtrl); - }, delay * 1000); + pollLineage(provenanceTableCtrl); + }, 2000); } }; @@ -1432,7 +1387,7 @@ nf.ng.ProvenanceLineage = function () { lineage = response.lineage; // process the results, if they are not done wait 1 second before trying again - processLineage(1, provenanceTableCtrl); + processLineage(provenanceTableCtrl); }).fail(closeDialog); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js index 89440ecc39..7da859317e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js @@ -34,7 +34,7 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { searchOptions: '../nifi-api/provenance/search-options', replays: '../nifi-api/provenance-events/replays', provenance: '../nifi-api/provenance', - provenanceEvents: '../nifi-api/provenance-events', + provenanceEvents: '../nifi-api/provenance-events/', clusterSearch: '../nifi-api/flow/cluster/search-results', d3Script: 'js/d3/d3.min.js', lineageScript: 'js/nf/provenance/nf-provenance-lineage.js', @@ -57,7 +57,7 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { var eventId = $('#provenance-event-id').text(); // build the url - var dataUri = config.urls.provenanceEvents + '/' + encodeURIComponent(eventId) + '/content/' + encodeURIComponent(direction); + var dataUri = config.urls.provenanceEvents + encodeURIComponent(eventId) + '/content/' + encodeURIComponent(direction); // perform the request once we've received a token nf.Common.getAccessToken(config.urls.downloadToken).done(function (downloadToken) { @@ -736,7 +736,7 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { } } else if (provenanceGrid.getColumns()[args.cell].id === 'moreDetails') { if (target.hasClass('show-event-details')) { - provenanceTableCtrl.showEventDetails(item); + provenanceTableCtrl.showEventDetails(item.eventId, item.clusterNodeId); } } }); @@ -864,7 +864,9 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { var provenanceEntity = { 'provenance': { 'request': $.extend({ - maxResults: config.maxResults + maxResults: config.maxResults, + summarize: true, + incrementalResults: false }, provenance) } }; @@ -889,7 +891,14 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { var url = provenance.uri; if (nf.Common.isDefinedAndNotNull(provenance.request.clusterNodeId)) { url += '?' + $.param({ - clusterNodeId: provenance.request.clusterNodeId + clusterNodeId: provenance.request.clusterNodeId, + summarize: true, + incrementalResults: false + }); + } else { + url += '?' + $.param({ + summarize: true, + incrementalResults: false }); } @@ -1003,11 +1012,6 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { function ProvenanceTableCtrl() { - /** - * The max delay between requests. - */ - this.MAX_DELAY = 4; - /** * The server time offset */ @@ -1147,21 +1151,19 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { $('#provenance-query-dialog').modal('hide'); }; - // polls the server for the status of the provenance, if the provenance is not - // done wait nextDelay seconds before trying again - var pollProvenance = function (nextDelay) { + // polls the server for the status of the provenance + var pollProvenance = function () { getProvenance(provenance).done(function (response) { // update the provenance provenance = response.provenance; // process the provenance - processProvenanceResponse(nextDelay); + processProvenanceResponse(); }).fail(closeDialog); }; - // processes the provenance, if the provenance is not done wait delay - // before polling again - var processProvenanceResponse = function (delay) { + // processes the provenance + var processProvenanceResponse = function () { // if the request was cancelled just ignore the current response if (cancelled === true) { closeDialog(); @@ -1193,13 +1195,9 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { // clear the timer since we've been invoked provenanceTimer = null; - // calculate the next delay (back off) - var backoff = delay * 2; - var nextDelay = backoff > self.MAX_DELAY ? self.MAX_DELAY : backoff; - // poll provenance - pollProvenance(nextDelay); - }, delay * 1000); + pollProvenance(); + }, 2000); } }; @@ -1209,223 +1207,251 @@ nf.ng.ProvenanceTable = function (provenanceLineageCtrl) { provenance = response.provenance; // process the results, if they are not done wait 1 second before trying again - processProvenanceResponse(1); + processProvenanceResponse(); }).fail(closeDialog); }, + /** + * Gets the details for the specified event. + * + * @param {string} eventId + * @param {string} clusterNodeId The id of the node in the cluster where this event/flowfile originated + */ + getEventDetails: function (eventId, clusterNodeId) { + var url; + if (nf.Common.isDefinedAndNotNull(clusterNodeId)) { + url = config.urls.provenanceEvents + encodeURIComponent(eventId) + '?' + $.param({ + clusterNodeId: clusterNodeId + }); + } else { + url = config.urls.provenanceEvents + encodeURIComponent(eventId); + } + + return $.ajax({ + type: 'GET', + url: url, + dataType: 'json' + }).fail(nf.Common.handleAjaxError); + }, + /** * Shows the details for the specified action. * - * @param {object} event + * @param {string} eventId + * @param {string} clusterNodeId The id of the node in the cluster where this event/flowfile originated */ - showEventDetails: function (event) { - // update the event details - $('#provenance-event-id').text(event.eventId); - $('#provenance-event-time').html(nf.Common.formatValue(event.eventTime)).ellipsis(); - $('#provenance-event-type').html(nf.Common.formatValue(event.eventType)).ellipsis(); - $('#provenance-event-flowfile-uuid').html(nf.Common.formatValue(event.flowFileUuid)).ellipsis(); - $('#provenance-event-component-id').html(nf.Common.formatValue(event.componentId)).ellipsis(); - $('#provenance-event-component-name').html(nf.Common.formatValue(event.componentName)).ellipsis(); - $('#provenance-event-component-type').html(nf.Common.formatValue(event.componentType)).ellipsis(); - $('#provenance-event-details').html(nf.Common.formatValue(event.details)).ellipsis(); + showEventDetails: function (eventId, clusterNodeId) { + provenanceTableCtrl.getEventDetails(eventId, clusterNodeId).done(function (response) { + var event = response.provenanceEvent; - // over the default tooltip with the actual byte count - var fileSize = $('#provenance-event-file-size').html(nf.Common.formatValue(event.fileSize)).ellipsis(); - fileSize.attr('title', nf.Common.formatInteger(event.fileSizeBytes) + ' bytes'); + // update the event details + $('#provenance-event-id').text(event.eventId); + $('#provenance-event-time').html(nf.Common.formatValue(event.eventTime)).ellipsis(); + $('#provenance-event-type').html(nf.Common.formatValue(event.eventType)).ellipsis(); + $('#provenance-event-flowfile-uuid').html(nf.Common.formatValue(event.flowFileUuid)).ellipsis(); + $('#provenance-event-component-id').html(nf.Common.formatValue(event.componentId)).ellipsis(); + $('#provenance-event-component-name').html(nf.Common.formatValue(event.componentName)).ellipsis(); + $('#provenance-event-component-type').html(nf.Common.formatValue(event.componentType)).ellipsis(); + $('#provenance-event-details').html(nf.Common.formatValue(event.details)).ellipsis(); - // sets an duration - var setDuration = function (field, value) { - if (nf.Common.isDefinedAndNotNull(value)) { - if (value === 0) { - field.text('< 1ms'); + // over the default tooltip with the actual byte count + var fileSize = $('#provenance-event-file-size').html(nf.Common.formatValue(event.fileSize)).ellipsis(); + fileSize.attr('title', nf.Common.formatInteger(event.fileSizeBytes) + ' bytes'); + + // sets an duration + var setDuration = function (field, value) { + if (nf.Common.isDefinedAndNotNull(value)) { + if (value === 0) { + field.text('< 1ms'); + } else { + field.text(nf.Common.formatDuration(value)); + } } else { - field.text(nf.Common.formatDuration(value)); + field.html('No value set'); + } + }; + + // handle durations + setDuration($('#provenance-event-duration'), event.eventDuration); + setDuration($('#provenance-lineage-duration'), event.lineageDuration); + + // formats an event detail + var formatEventDetail = function (label, value) { + $('
').append( + $('
').text(label)).append( + $('
' + nf.Common.formatValue(value) + '
').ellipsis()).append( + $('
')).appendTo('#additional-provenance-details'); + }; + + // conditionally show RECEIVE details + if (event.eventType === 'RECEIVE') { + formatEventDetail('Source FlowFile Id', event.sourceSystemFlowFileId); + formatEventDetail('Transit Uri', event.transitUri); + } + + // conditionally show SEND details + if (event.eventType === 'SEND') { + formatEventDetail('Transit Uri', event.transitUri); + } + + // conditionally show ADDINFO details + if (event.eventType === 'ADDINFO') { + formatEventDetail('Alternate Identifier Uri', event.alternateIdentifierUri); + } + + // conditionally show ROUTE details + if (event.eventType === 'ROUTE') { + formatEventDetail('Relationship', event.relationship); + } + + // conditionally show FETCH details + if (event.eventType === 'FETCH') { + formatEventDetail('Transit Uri', event.transitUri); + } + + // conditionally show the cluster node identifier + if (nf.Common.isDefinedAndNotNull(event.clusterNodeId)) { + // save the cluster node id + $('#provenance-event-cluster-node-id').text(event.clusterNodeId); + + // render the cluster node address + formatEventDetail('Node Address', event.clusterNodeAddress); + } + + // populate the parent/child flowfile uuids + var parentUuids = $('#parent-flowfiles-container'); + var childUuids = $('#child-flowfiles-container'); + + // handle parent flowfiles + if (nf.Common.isEmpty(event.parentUuids)) { + $('#parent-flowfile-count').text(0); + parentUuids.append('No parents'); + } else { + $('#parent-flowfile-count').text(event.parentUuids.length); + $.each(event.parentUuids, function (_, uuid) { + $('
').text(uuid).appendTo(parentUuids); + }); + } + + // handle child flowfiles + if (nf.Common.isEmpty(event.childUuids)) { + $('#child-flowfile-count').text(0); + childUuids.append('No children'); + } else { + $('#child-flowfile-count').text(event.childUuids.length); + $.each(event.childUuids, function (_, uuid) { + $('
').text(uuid).appendTo(childUuids); + }); + } + + // get the attributes container + var attributesContainer = $('#attributes-container'); + + // get any action details + $.each(event.attributes, function (_, attribute) { + // create the attribute record + var attributeRecord = $('
') + .append($('
' + nf.Common.formatValue(attribute.name) + '
').ellipsis()) + .appendTo(attributesContainer); + + // add the current value + attributeRecord + .append($('
' + nf.Common.formatValue(attribute.value) + '
').ellipsis()) + .append('
'); + + // show the previous value if the property has changed + if (attribute.value !== attribute.previousValue) { + if (nf.Common.isDefinedAndNotNull(attribute.previousValue)) { + attributeRecord + .append($('
' + nf.Common.formatValue(attribute.previousValue) + ' (previous)
').ellipsis()) + .append('
'); + } else { + attributeRecord + .append($('
' + nf.Common.formatValue(attribute.previousValue) + '
').ellipsis()) + .append('
'); + } + } else { + // mark this attribute as not modified + attributeRecord.addClass('attribute-unmodified'); + } + }); + + var formatContentValue = function (element, value) { + if (nf.Common.isDefinedAndNotNull(value)) { + element.removeClass('unset').text(value); + } else { + element.addClass('unset').text('No value previously set'); + } + }; + + // content + $('#input-content-header').text('Input Claim'); + formatContentValue($('#input-content-container'), event.inputContentClaimContainer); + formatContentValue($('#input-content-section'), event.inputContentClaimSection); + formatContentValue($('#input-content-identifier'), event.inputContentClaimIdentifier); + formatContentValue($('#input-content-offset'), event.inputContentClaimOffset); + formatContentValue($('#input-content-bytes'), event.inputContentClaimFileSizeBytes); + + // input content file size + var inputContentSize = $('#input-content-size'); + formatContentValue(inputContentSize, event.inputContentClaimFileSize); + if (nf.Common.isDefinedAndNotNull(event.inputContentClaimFileSize)) { + // over the default tooltip with the actual byte count + inputContentSize.attr('title', nf.Common.formatInteger(event.inputContentClaimFileSizeBytes) + ' bytes'); + } + + formatContentValue($('#output-content-container'), event.outputContentClaimContainer); + formatContentValue($('#output-content-section'), event.outputContentClaimSection); + formatContentValue($('#output-content-identifier'), event.outputContentClaimIdentifier); + formatContentValue($('#output-content-offset'), event.outputContentClaimOffset); + formatContentValue($('#output-content-bytes'), event.outputContentClaimFileSizeBytes); + + // output content file size + var outputContentSize = $('#output-content-size'); + formatContentValue(outputContentSize, event.outputContentClaimFileSize); + if (nf.Common.isDefinedAndNotNull(event.outputContentClaimFileSize)) { + // over the default tooltip with the actual byte count + outputContentSize.attr('title', nf.Common.formatInteger(event.outputContentClaimFileSizeBytes) + ' bytes'); + } + + if (event.inputContentAvailable === true) { + $('#input-content-download').show(); + + if (nf.Common.isContentViewConfigured()) { + $('#input-content-view').show(); + } else { + $('#input-content-view').hide(); } } else { - field.html('No value set'); - } - }; - - // handle durations - setDuration($('#provenance-event-duration'), event.eventDuration); - setDuration($('#provenance-lineage-duration'), event.lineageDuration); - - // formats an event detail - var formatEventDetail = function (label, value) { - $('
').append( - $('
').text(label)).append( - $('
' + nf.Common.formatValue(value) + '
').ellipsis()).append( - $('
')).appendTo('#additional-provenance-details'); - }; - - // conditionally show RECEIVE details - if (event.eventType === 'RECEIVE') { - formatEventDetail('Source FlowFile Id', event.sourceSystemFlowFileId); - formatEventDetail('Transit Uri', event.transitUri); - } - - // conditionally show SEND details - if (event.eventType === 'SEND') { - formatEventDetail('Transit Uri', event.transitUri); - } - - // conditionally show ADDINFO details - if (event.eventType === 'ADDINFO') { - formatEventDetail('Alternate Identifier Uri', event.alternateIdentifierUri); - } - - // conditionally show ROUTE details - if (event.eventType === 'ROUTE') { - formatEventDetail('Relationship', event.relationship); - } - - // conditionally show FETCH details - if (event.eventType === 'FETCH') { - formatEventDetail('Transit Uri', event.transitUri); - } - - // conditionally show the cluster node identifier - if (nf.Common.isDefinedAndNotNull(event.clusterNodeId)) { - // save the cluster node id - $('#provenance-event-cluster-node-id').text(event.clusterNodeId); - - // render the cluster node address - formatEventDetail('Node Address', event.clusterNodeAddress); - } - - // populate the parent/child flowfile uuids - var parentUuids = $('#parent-flowfiles-container'); - var childUuids = $('#child-flowfiles-container'); - - // handle parent flowfiles - if (nf.Common.isEmpty(event.parentUuids)) { - $('#parent-flowfile-count').text(0); - parentUuids.append('No parents'); - } else { - $('#parent-flowfile-count').text(event.parentUuids.length); - $.each(event.parentUuids, function (_, uuid) { - $('
').text(uuid).appendTo(parentUuids); - }); - } - - // handle child flowfiles - if (nf.Common.isEmpty(event.childUuids)) { - $('#child-flowfile-count').text(0); - childUuids.append('No children'); - } else { - $('#child-flowfile-count').text(event.childUuids.length); - $.each(event.childUuids, function (_, uuid) { - $('
').text(uuid).appendTo(childUuids); - }); - } - - // get the attributes container - var attributesContainer = $('#attributes-container'); - - // get any action details - $.each(event.attributes, function (_, attribute) { - // create the attribute record - var attributeRecord = $('
') - .append($('
' + nf.Common.formatValue(attribute.name) + '
').ellipsis()) - .appendTo(attributesContainer); - - // add the current value - attributeRecord - .append($('
' + nf.Common.formatValue(attribute.value) + '
').ellipsis()) - .append('
'); - - // show the previous value if the property has changed - if (attribute.value !== attribute.previousValue) { - if (nf.Common.isDefinedAndNotNull(attribute.previousValue)) { - attributeRecord - .append($('
' + nf.Common.formatValue(attribute.previousValue) + ' (previous)
').ellipsis()) - .append('
'); - } else { - attributeRecord - .append($('
' + nf.Common.formatValue(attribute.previousValue) + '
').ellipsis()) - .append('
'); - } - } else { - // mark this attribute as not modified - attributeRecord.addClass('attribute-unmodified'); - } - }); - - var formatContentValue = function (element, value) { - if (nf.Common.isDefinedAndNotNull(value)) { - element.removeClass('unset').text(value); - } else { - element.addClass('unset').text('No value previously set'); - } - }; - - // content - $('#input-content-header').text('Input Claim'); - formatContentValue($('#input-content-container'), event.inputContentClaimContainer); - formatContentValue($('#input-content-section'), event.inputContentClaimSection); - formatContentValue($('#input-content-identifier'), event.inputContentClaimIdentifier); - formatContentValue($('#input-content-offset'), event.inputContentClaimOffset); - formatContentValue($('#input-content-bytes'), event.inputContentClaimFileSizeBytes); - - // input content file size - var inputContentSize = $('#input-content-size'); - formatContentValue(inputContentSize, event.inputContentClaimFileSize); - if (nf.Common.isDefinedAndNotNull(event.inputContentClaimFileSize)) { - // over the default tooltip with the actual byte count - inputContentSize.attr('title', nf.Common.formatInteger(event.inputContentClaimFileSizeBytes) + ' bytes'); - } - - formatContentValue($('#output-content-container'), event.outputContentClaimContainer); - formatContentValue($('#output-content-section'), event.outputContentClaimSection); - formatContentValue($('#output-content-identifier'), event.outputContentClaimIdentifier); - formatContentValue($('#output-content-offset'), event.outputContentClaimOffset); - formatContentValue($('#output-content-bytes'), event.outputContentClaimFileSizeBytes); - - // output content file size - var outputContentSize = $('#output-content-size'); - formatContentValue(outputContentSize, event.outputContentClaimFileSize); - if (nf.Common.isDefinedAndNotNull(event.outputContentClaimFileSize)) { - // over the default tooltip with the actual byte count - outputContentSize.attr('title', nf.Common.formatInteger(event.outputContentClaimFileSizeBytes) + ' bytes'); - } - - if (event.inputContentAvailable === true) { - $('#input-content-download').show(); - - if (nf.Common.isContentViewConfigured()) { - $('#input-content-view').show(); - } else { + $('#input-content-download').hide(); $('#input-content-view').hide(); } - } else { - $('#input-content-download').hide(); - $('#input-content-view').hide(); - } - if (event.outputContentAvailable === true) { - $('#output-content-download').show(); + if (event.outputContentAvailable === true) { + $('#output-content-download').show(); - if (nf.Common.isContentViewConfigured()) { - $('#output-content-view').show(); + if (nf.Common.isContentViewConfigured()) { + $('#output-content-view').show(); + } else { + $('#output-content-view').hide(); + } } else { + $('#output-content-download').hide(); $('#output-content-view').hide(); } - } else { - $('#output-content-download').hide(); - $('#output-content-view').hide(); - } - if (event.replayAvailable === true) { - $('#replay-content, #replay-content-connection').show(); - formatContentValue($('#replay-connection-id'), event.sourceConnectionIdentifier); - $('#replay-content-message').hide(); - } else { - $('#replay-content, #replay-content-connection').hide(); - $('#replay-content-message').text(event.replayExplanation).show(); - } + if (event.replayAvailable === true) { + $('#replay-content, #replay-content-connection').show(); + formatContentValue($('#replay-connection-id'), event.sourceConnectionIdentifier); + $('#replay-content-message').hide(); + } else { + $('#replay-content, #replay-content-connection').hide(); + $('#replay-content-message').text(event.replayExplanation).show(); + } - // show the dialog - $('#event-details-dialog').modal('show'); + // show the dialog + $('#event-details-dialog').modal('show'); + }); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js index 9526ff8348..9e5d457f8b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance.js @@ -83,7 +83,7 @@ nf.ng.Provenance = function (provenanceTableCtrl) { */ var loadAbout = function () { // get the about details - $.ajax({ + return $.ajax({ type: 'GET', url: config.urls.about, dataType: 'json'