Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-14141 Enabled Expression Language for File Size in GenerateFlowFile (#9620) 

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
markobean 2025-01-10 16:53:37 -05:00 committed by GitHub
parent 58cce330d5
commit 7899874e88
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 63 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GenerateFlowFile.java
View File

@ -77,6 +77,7 @@ public class GenerateFlowFile extends AbstractProcessor {
.required(true)
.defaultValue("0B")
.addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
.build();
public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
.name("Batch Size")
@ -191,7 +192,7 @@ public class GenerateFlowFile extends AbstractProcessor {
}
private byte[] generateData(final ProcessContext context) {
final int byteCount = context.getProperty(FILE_SIZE).asDataSize(DataUnit.B).intValue();
final int byteCount = context.getProperty(FILE_SIZE).evaluateAttributeExpressions().asDataSize(DataUnit.B).intValue();
final Random random = new Random();
final byte[] array = new byte[byteCount];

31
nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.GenerateFlowFile/additionalDetails.md Normal file
View File

@ -0,0 +1,31 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# GenerateFlowFile
This processor can be configured to generate variable-sized FlowFiles. The `File Size` property accepts both a literal
value, e.g. "1 KB", and Expression Language statements. In order to create FlowFiles of variable sizes, the Expression
Language function `random()` can be used. For example, `${random():mod(101)}` will generate values between 0 and 100,
inclusive. A data size label, e.g. B, KB, MB, etc., must be included in the Expression Language statement since the
`File Size` property holds a data size value. The table below shows some examples.
| File Size Expression Language Statement | File Sizes Generated (values are inclusive) |
|--------------------------------------------|---------------------------------------------|
| ${random():mod(101)}b | 0 - 100 bytes |
| ${random():mod(101)}mb | 0 - 100 MB |
| ${random():mod(101):plus(20)} B | 20 - 120 bytes |
| ${random():mod(71):plus(30):append("KB")} | 30 - 100 KB |
See the Expression Language Guide for more details on the `random()` function.

30
nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGenerateFlowFile.java
View File

@ -21,6 +21,10 @@ import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Unit tests for the GenerateFlowFile processor.
*/
@ -73,4 +77,30 @@ public class TestGenerateFlowFile {
generatedFlowFile.assertAttributeEquals("mime.type", "application/text");
}
@Test
public void testExpressionLanguageSupport() {
TestRunner runner = TestRunners.newTestRunner(new GenerateFlowFile());
runner.setProperty(GenerateFlowFile.FILE_SIZE, "${nextInt()}B");
runner.setProperty(GenerateFlowFile.UNIQUE_FLOWFILES, "true");
runner.setProperty(GenerateFlowFile.BATCH_SIZE, "2");
runner.assertValid();
runner.run();
// verify multiple files in a batch each have a unique file size based on the given Expression Language and uniqueness set to true
runner.assertTransferCount(GenerateFlowFile.SUCCESS, 2);
assertTrue(runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(0).getSize() < runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(1).getSize());
runner.clearTransferState();
runner.setProperty(GenerateFlowFile.UNIQUE_FLOWFILES, "false");
runner.assertValid();
runner.run();
// verify multiple files in a batch each have the same file size when uniqueness is set to false
runner.assertTransferCount(GenerateFlowFile.SUCCESS, 2);
assertEquals(runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(0).getSize(), runner.getFlowFilesForRelationship(GenerateFlowFile.SUCCESS).get(1).getSize());
}
}