From 7899874e881125029c42be503ac1d028d87bf995 Mon Sep 17 00:00:00 2001 From: markobean Date: Fri, 10 Jan 2025 16:53:37 -0500 Subject: [PATCH] NIFI-14141 Enabled Expression Language for File Size in GenerateFlowFile (#9620) Signed-off-by: David Handermann --- .../processors/standard/GenerateFlowFile.java | 3 +- .../additionalDetails.md | 31 +++++++++++++++++++ .../standard/TestGenerateFlowFile.java | 30 ++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.GenerateFlowFile/additionalDetails.md diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java index 257c1d244a..9031e7662e 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java @@ -77,6 +77,7 @@ public class GenerateFlowFile extends AbstractProcessor { .required(true) .defaultValue("0B") .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) .build(); public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() .name("Batch Size") @@ -191,7 +192,7 @@ public class GenerateFlowFile extends AbstractProcessor { } private byte[] generateData(final ProcessContext context) { - final int byteCount = context.getProperty(FILE_SIZE).asDataSize(DataUnit.B).intValue(); + final int byteCount = context.getProperty(FILE_SIZE).evaluateAttributeExpressions().asDataSize(DataUnit.B).intValue(); final Random random = new Random(); final byte[] array = new byte[byteCount]; diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.GenerateFlowFile/additionalDetails.md b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.GenerateFlowFile/additionalDetails.md new file mode 100644 index 0000000000..41a852a910 --- /dev/null +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.GenerateFlowFile/additionalDetails.md @@ -0,0 +1,31 @@ + + +# GenerateFlowFile + +This processor can be configured to generate variable-sized FlowFiles. The `File Size` property accepts both a literal +value, e.g. "1 KB", and Expression Language statements. In order to create FlowFiles of variable sizes, the Expression +Language function `random()` can be used. For example, `${random():mod(101)}` will generate values between 0 and 100, +inclusive. A data size label, e.g. B, KB, MB, etc., must be included in the Expression Language statement since the +`File Size` property holds a data size value. The table below shows some examples. + +| File Size Expression Language Statement | File Sizes Generated (values are inclusive) | +|--------------------------------------------|---------------------------------------------| +| ${random():mod(101)}b | 0 - 100 bytes | +| ${random():mod(101)}mb | 0 - 100 MB | +| ${random():mod(101):plus(20)} B | 20 - 120 bytes | +| ${random():mod(71):plus(30):append("KB")} | 30 - 100 KB | + +See the Expression Language Guide for more details on the `random()` function. \ No newline at end of file diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java index 138cb682a9..1ee7018ac2 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java @@ -21,6 +21,10 @@ import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + /** * Unit tests for the GenerateFlowFile processor. */ @@ -73,4 +77,30 @@ public class TestGenerateFlowFile { generatedFlowFile.assertAttributeEquals("mime.type", "application/text"); } + @Test + public void testExpressionLanguageSupport() { + TestRunner runner = TestRunners.newTestRunner(new GenerateFlowFile()); + runner.setProperty(GenerateFlowFile.FILE_SIZE, "${nextInt()}B"); + runner.setProperty(GenerateFlowFile.UNIQUE_FLOWFILES, "true"); + runner.setProperty(GenerateFlowFile.BATCH_SIZE, "2"); + runner.assertValid(); + + runner.run(); + + // verify multiple files in a batch each have a unique file size based on the given Expression Language and uniqueness set to true + runner.assertTransferCount(GenerateFlowFile.SUCCESS, 2); + assertTrue(runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(0).getSize() < runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(1).getSize()); + runner.clearTransferState(); + + runner.setProperty(GenerateFlowFile.UNIQUE_FLOWFILES, "false"); + runner.assertValid(); + + runner.run(); + + // verify multiple files in a batch each have the same file size when uniqueness is set to false + runner.assertTransferCount(GenerateFlowFile.SUCCESS, 2); + assertEquals(runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(0).getSize(), runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(1).getSize()); + } + + } \ No newline at end of file