From 027e2b9bc107843a690427c3960a14599b873957 Mon Sep 17 00:00:00 2001 From: Kalman Jantner Date: Fri, 27 Jan 2023 16:41:50 +0100 Subject: [PATCH] NIFI-11123: fix default value and update docs This closes #6899. Signed-off-by: Tamas Palfy --- .../AbstractStartGcpVisionOperation.java | 39 ++++++++++++++++++- .../StartGcpVisionAnnotateFilesOperation.java | 6 +-- ...StartGcpVisionAnnotateImagesOperation.java | 6 +-- .../additionalDetails.html | 11 ++++-- .../additionalDetails.html | 15 +++++-- 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/AbstractStartGcpVisionOperation.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/AbstractStartGcpVisionOperation.java index 373d933b7e..a24beb4558 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/AbstractStartGcpVisionOperation.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/AbstractStartGcpVisionOperation.java @@ -24,14 +24,40 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; public abstract class AbstractStartGcpVisionOperation> extends AbstractGcpVisionProcessor { + public static final PropertyDescriptor FEATURE_TYPE = new PropertyDescriptor.Builder() + .name("vision-feature-type") + .displayName("Vision Feature Type") + .description("Type of GCP Vision Feature. The value of this property applies when the JSON Payload property is configured. " + + "The JSON Payload property value can use Expression Language to reference the value of ${vision-feature-type}") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .defaultValue("TEXT_DETECTION") + .build(); + public static final PropertyDescriptor OUTPUT_BUCKET = new PropertyDescriptor.Builder() + .name("output-bucket") + .displayName("Output Bucket") + .description("Name of the GCS bucket where the output of the Vision job will be persisted. " + + "The value of this property applies when the JSON Payload property is configured. " + + "The JSON Payload property value can use Expression Language to reference the value of ${output-bucket}") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) + .build(); + @Override public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { @@ -70,7 +96,18 @@ public abstract class AbstractStartGcpVisionOperation attributes = new HashMap<>(); + attributes.put(OUTPUT_BUCKET.getName(), getAttributeValue(context, flowFile, OUTPUT_BUCKET.getName())); + attributes.put(FEATURE_TYPE.getName(), getAttributeValue(context, flowFile, FEATURE_TYPE.getName())); + final PropertyValue jsonPropertyValue = context.getProperty(getJsonPayloadPropertyDescriptor()); + final String jsonPayload = jsonPropertyValue.evaluateAttributeExpressions(flowFile, attributes).getValue(); + return new ByteArrayInputStream(jsonPayload.getBytes(StandardCharsets.UTF_8)); + } + + private String getAttributeValue(ProcessContext context, FlowFile flowFile, String name) { + final String flowFileAttribute = flowFile.getAttribute(name); + final PropertyValue propertyValue = context.getProperty(name); + return flowFileAttribute == null ? propertyValue.getValue() : flowFileAttribute; } abstract B newBuilder(); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateFilesOperation.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateFilesOperation.java index 5c7014f722..e79ab606a0 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateFilesOperation.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateFilesOperation.java @@ -56,12 +56,12 @@ public class StartGcpVisionAnnotateFilesOperation extends AbstractStartGcpVision " \"mimeType\": \"application/pdf\"\n" + " },\n" + " \"features\": [{\n" + - " \"type\": \"DOCUMENT_TEXT_DETECTION\",\n" + + " \"type\": \"${vision-feature-type}\",\n" + " \"maxResults\": 4\n" + " }],\n" + " \"outputConfig\": {\n" + " \"gcsDestination\": {\n" + - " \"uri\": \"gs://${gcs.bucket}/${filename}/\"\n" + + " \"uri\": \"gs://${output-bucket}/${filename}/\"\n" + " },\n" + " \"batchSize\": 2\n" + " }\n" + @@ -70,7 +70,7 @@ public class StartGcpVisionAnnotateFilesOperation extends AbstractStartGcpVision .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); private static final List PROPERTIES = Collections.unmodifiableList(Arrays.asList( - JSON_PAYLOAD, GCP_CREDENTIALS_PROVIDER_SERVICE)); + JSON_PAYLOAD, GCP_CREDENTIALS_PROVIDER_SERVICE, OUTPUT_BUCKET, FEATURE_TYPE)); @Override public List getSupportedPropertyDescriptors() { diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateImagesOperation.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateImagesOperation.java index 6d078ec69b..41510377f9 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateImagesOperation.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/vision/StartGcpVisionAnnotateImagesOperation.java @@ -55,13 +55,13 @@ public class StartGcpVisionAnnotateImagesOperation extends AbstractStartGcpVisio " }\n" + " },\n" + " \"features\": [{\n" + - " \"type\": \"FACE_DETECTION\",\n" + + " \"type\": \"${vision-feature-type}\",\n" + " \"maxResults\": 4\n" + " }]\n" + " }],\n" + " \"outputConfig\": {\n" + " \"gcsDestination\": {\n" + - " \"uri\": \"gs://${gcs.bucket}/${filename}/\"\n" + + " \"uri\": \"gs://${output-bucket}/${filename}/\"\n" + " },\n" + " \"batchSize\": 2\n" + " }\n" + @@ -69,7 +69,7 @@ public class StartGcpVisionAnnotateImagesOperation extends AbstractStartGcpVisio .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); private static final List PROPERTIES = Collections.unmodifiableList(Arrays.asList( - JSON_PAYLOAD, GCP_CREDENTIALS_PROVIDER_SERVICE)); + JSON_PAYLOAD, GCP_CREDENTIALS_PROVIDER_SERVICE, OUTPUT_BUCKET, FEATURE_TYPE)); @Override public List getSupportedPropertyDescriptors() { diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateFilesOperation/additionalDetails.html b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateFilesOperation/additionalDetails.html index fe99aefff2..fded578178 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateFilesOperation/additionalDetails.html +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateFilesOperation/additionalDetails.html @@ -55,12 +55,12 @@ "mimeType": "application/pdf" }, "features": [{ - "type": "DOCUMENT_TEXT_DETECTION", + "type": "${vision-feature-type}", "maxResults": 4 }], "outputConfig": { "gcsDestination": { - "uri": "gs://${gcs.bucket}/${filename}/" + "uri": "gs://${output-bucket}/${filename}/" }, "batchSize": 2 } @@ -88,7 +88,12 @@ You can find more details at

- Keep the default value of JSON PAYLOAD property in StartGcpVisionAnnotateImagesOperation +

  • Create an input and output bucket
    • +
  • Make sure the input files(s) are available in the input GCS bucket
    • +
  • Set the bucket property of ListGCSBucket processor to your input bucket name
    • +
  • Keep the default value of JSON PAYLOAD property in StartGcpVisionAnnotateFilesOperation
    • +
  • Set the Output Bucket property to your output bucket name in StartGcpVisionAnnotateFilesOperation
    • +
  • Setup GCP Credentials Provider Service for all GCP related processor

    • Execution steps: diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateImagesOperation/additionalDetails.html b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateImagesOperation/additionalDetails.html index e15796fcb0..52a2800193 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateImagesOperation/additionalDetails.html +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/resources/docs/org.apache.nifi.processors.gcp.vision.StartGcpVisionAnnotateImagesOperation/additionalDetails.html @@ -27,7 +27,7 @@ Prerequisites

    • Make sure Vision API is enabled and the account you are using has the right to use it
      • -
    • Make sure thne input image(s) are available in a GCS bucket
      • +
    • Make sure the input image(s) are available in a GCS bucket under /input folder

    Usage

    @@ -53,13 +53,13 @@ } }, "features": [{ - "type": "DOCUMENT_TEXT_DETECTION", + "type": "${vision-feature-type}", "maxResults": 4 }] }], "outputConfig": { "gcsDestination": { - "uri": "gs://${gcs.bucket}/${filename}/" + "uri": "gs://${output-bucket}/${filename}/" }, "batchSize": 2 } @@ -89,7 +89,14 @@ You can find more details at

    - Keep the default value of JSON PAYLOAD property in StartGcpVisionAnnotateImagesOperation +

      +
    • Create an input and output bucket
      • +
    • Make sure the input images(s) are available in the input GCS bucket
      • +
    • Set the bucket property of ListGCSBucket processor to your input bucket name
      • +
    • Keep the default value of JSON PAYLOAD property in StartGcpVisionAnnotateImagesOperation
      • +
    • Set the Output Bucket property to your output bucket name in StartGcpVisionAnnotateImagesOperation
      • +
    • Setup GCP Credentials Provider Service for all GCP related processor
      • +

    Execution steps: