Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-12781 Removed File Location from Provenance UPLOAD event 

This closes #8397

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
lehelb 2024-02-12 15:49:37 -05:00 committed by exceptionfactory
parent 14fcc42d16
commit d1af7ef0c5
No known key found for this signature in database
4 changed files with 16 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceFileResource.java
View File

@ -1,29 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.provenance;
/**
* Holds information of a file resource for UPLOAD
* provenance events.
*/
public record ProvenanceFileResource(String location, long size) {
@Override
public String toString() {
return "ProvenanceFileResource[location=%s, size=%d]".formatted(location, size);
}
}

8
nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceReporter.java
View File

@ -323,14 +323,14 @@ public interface ProvenanceReporter {
* local destination, such as the local file system but is external to NiFi.
*
* @param flowFile the FlowFile that was sent
* @param fileResource the FileResource that was uploaded
* @param size the size of the resource in bytes that was uploaded
* @param transitUri A URI that provides information about the System and
* Protocol information over which the transfer occurred. The intent of this
* field is such that both the sender and the receiver can publish the
* events to an external Enterprise-wide system that is then able to
* correlate the SEND and RECEIVE events.
*/
void upload(FlowFile flowFile, ProvenanceFileResource fileResource, String transitUri);
void upload(FlowFile flowFile, long size, String transitUri);
/**
* Emits a Provenance Event of type {@link ProvenanceEventType#UPLOAD UPLOAD}
@ -339,7 +339,7 @@ public interface ProvenanceReporter {
* local destination, such as the local file system but is external to NiFi.
*
* @param flowFile the FlowFile that was sent
* @param fileResource the FileResource that was uploaded
* @param size the size of the resource in bytes that was uploaded
* @param transitUri A URI that provides information about the System and
* Protocol information over which the transfer occurred. The intent of this
* field is such that both the sender and the receiver can publish the
@ -355,7 +355,7 @@ public interface ProvenanceReporter {
* ProvenanceReporter is associated is rolled back. Otherwise, the Event
* will be recorded only on a successful session commit.
*/
void upload(FlowFile flowFile, ProvenanceFileResource fileResource, String transitUri, String details, long transmissionMillis, boolean force);
void upload(FlowFile flowFile, long size, String transitUri, String details, long transmissionMillis, boolean force);
/**
* Emits a Provenance Event of type {@link ProvenanceEventType#REMOTE_INVOCATION}

12
nifi-mock/src/main/java/org/apache/nifi/util/MockProvenanceReporter.java
View File

@ -24,7 +24,6 @@ import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.provenance.ProvenanceReporter;
import org.apache.nifi.provenance.StandardProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceFileResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -228,16 +227,17 @@ public class MockProvenanceReporter implements ProvenanceReporter {
}
@Override
public void upload(final FlowFile flowFile, final ProvenanceFileResource fileResource, final String transitUri) {
upload(flowFile, fileResource, transitUri, null, -1L, true);
public void upload(final FlowFile flowFile, final long size, final String transitUri) {
upload(flowFile, size, transitUri, null, -1L, true);
}
@Override
public void upload(FlowFile flowFile, ProvenanceFileResource fileResource, String transitUri, String details, long transmissionMillis, boolean force) {
public void upload(final FlowFile flowFile, final long size, final String transitUri, final String details,
final long transmissionMillis, final boolean force) {
try {
final String fileResourceDetails = fileResource.toString();
final String enrichedDetails = details == null ? fileResourceDetails : details + " " + fileResourceDetails;
final String displayedSizeInBytes = size + " bytes";
final String enrichedDetails = details == null ? displayedSizeInBytes : details + " " + displayedSizeInBytes;
final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.UPLOAD)
.setTransitUri(transitUri)
.setEventDuration(transmissionMillis)

13
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProvenanceReporter.java
View File

@ -24,7 +24,6 @@ import org.apache.nifi.provenance.ProvenanceEventBuilder;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventRepository;
import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.provenance.ProvenanceFileResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -282,15 +281,15 @@ public class StandardProvenanceReporter implements InternalProvenanceReporter {
}
@Override
public void upload(final FlowFile flowFile, final ProvenanceFileResource fileResource, final String transitUri) {
upload(flowFile, fileResource, transitUri, null, -1L, true);
public void upload(final FlowFile flowFile, final long size, final String transitUri) {
upload(flowFile, size, transitUri, null, -1L, true);
}
@Override
public void upload(final FlowFile flowFile, final ProvenanceFileResource fileResource, final String transitUri, final String details, final long transmissionMillis, final boolean force) {
public void upload(final FlowFile flowFile, final long size, final String transitUri, final String details, final long transmissionMillis, final boolean force) {
try {
final String fileResourceDetails = fileResource.toString();
final String enrichedDetails = details == null ? fileResourceDetails : details + " " + fileResourceDetails;
final String displayedSizeInBytes = size + " bytes";
final String enrichedDetails = details == null ? displayedSizeInBytes : details + " " + displayedSizeInBytes;
final ProvenanceEventRecord record = build(flowFile, ProvenanceEventType.UPLOAD)
.setTransitUri(transitUri)
.setEventDuration(transmissionMillis)
@ -306,7 +305,7 @@ public class StandardProvenanceReporter implements InternalProvenanceReporter {
events.add(enriched);
}
bytesSent += fileResource.size();
bytesSent += size;
} catch (final Exception e) {
logger.error("Failed to generate Provenance Event", e);
}