Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-6964 This closes #3975. Use compression level for xz-lzma2 format of the CompressContent processor 

NIFI-6965 adding resource warning on memory and highlighting this with the compression level property

Signed-off-by: Joe Witt <joewitt@apache.org>
This commit is contained in:
John Pierce 2020-01-09 19:39:39 -05:00 committed by Joe Witt
parent b7fb94723c
commit 0dc6439ddb
No known key found for this signature in database
GPG Key ID: 9093BF854F811A1A
1 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
View File

@ -76,12 +76,14 @@ import org.xerial.snappy.SnappyOutputStream;
@InputRequirement(Requirement.INPUT_REQUIRED) @InputRequirement(Requirement.INPUT_REQUIRED)
@Tags({"content", "compress", "decompress", "gzip", "bzip2", "lzma", "xz-lzma2", "snappy", "snappy framed", "lz4-framed"}) @Tags({"content", "compress", "decompress", "gzip", "bzip2", "lzma", "xz-lzma2", "snappy", "snappy framed", "lz4-framed"})
@CapabilityDescription("Compresses or decompresses the contents of FlowFiles using a user-specified compression algorithm and updates the mime.type " @CapabilityDescription("Compresses or decompresses the contents of FlowFiles using a user-specified compression algorithm and updates the mime.type "
+ "attribute as appropriate") + "attribute as appropriate. This processor operates in a very memory efficient way so very large objects well beyond the heap size "
+ "are generally fine to process")
@ReadsAttribute(attribute = "mime.type", description = "If the Compression Format is set to use mime.type attribute, this attribute is used to " @ReadsAttribute(attribute = "mime.type", description = "If the Compression Format is set to use mime.type attribute, this attribute is used to "
+ "determine the compression type. Otherwise, this attribute is ignored.") + "determine the compression type. Otherwise, this attribute is ignored.")
@WritesAttribute(attribute = "mime.type", description = "If the Mode property is set to compress, the appropriate MIME Type is set. If the Mode " @WritesAttribute(attribute = "mime.type", description = "If the Mode property is set to compress, the appropriate MIME Type is set. If the Mode "
+ "property is set to decompress and the file is successfully decompressed, this attribute is removed, as the MIME Type is no longer known.") + "property is set to decompress and the file is successfully decompressed, this attribute is removed, as the MIME Type is no longer known.")
@SystemResourceConsideration(resource = SystemResource.CPU) @SystemResourceConsideration(resource = SystemResource.CPU)
@SystemResourceConsideration(resource = SystemResource.MEMORY)
public class CompressContent extends AbstractProcessor { public class CompressContent extends AbstractProcessor {
public static final String COMPRESSION_FORMAT_ATTRIBUTE = "use mime.type attribute"; public static final String COMPRESSION_FORMAT_ATTRIBUTE = "use mime.type attribute";
@ -107,8 +109,9 @@ public class CompressContent extends AbstractProcessor {
.build(); .build();
public static final PropertyDescriptor COMPRESSION_LEVEL = new PropertyDescriptor.Builder() public static final PropertyDescriptor COMPRESSION_LEVEL = new PropertyDescriptor.Builder()
.name("Compression Level") .name("Compression Level")
.description("The compression level to use; this is valid only when using GZIP compression. A lower value results in faster processing " .description("The compression level to use; this is valid only when using gzip or xz-lzma2 compression. A lower value results in faster processing "
+ "but less compression; a value of 0 indicates no compression but simply archiving") + "but less compression; a value of 0 indicates no (that is, simple archiving) for gzip or minimal for xz-lzma2 compression."
+ " Higher levels can mean much larger memory usage such as the case with levels 7-9 for xz-lzma/2 so be careful relative to heap size.")
.defaultValue("1") .defaultValue("1")
.required(true) .required(true)
.allowableValues("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") .allowableValues("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
@ -264,7 +267,8 @@ public class CompressContent extends AbstractProcessor {
mimeTypeRef.set("application/x-lzma"); mimeTypeRef.set("application/x-lzma");
break; break;
case COMPRESSION_FORMAT_XZ_LZMA2: case COMPRESSION_FORMAT_XZ_LZMA2:
compressionOut = new XZOutputStream(bufferedOut, new LZMA2Options()); final int xzCompressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
compressionOut = new XZOutputStream(bufferedOut, new LZMA2Options(xzCompressionLevel));
mimeTypeRef.set("application/x-xz"); mimeTypeRef.set("application/x-xz");
break; break;
case COMPRESSION_FORMAT_SNAPPY: case COMPRESSION_FORMAT_SNAPPY: