Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-11434 added support for optionally allowing zip entries with dat… (#7462) 

* NIFI-11434 added support for optionally allowing zip entries with data descriptors
* set default value and ensured all zip tests use various configs
This commit is contained in:
Joe Witt 2023-07-12 14:09:03 -07:00 committed by GitHub
parent 33e04795ef
commit d2c70d1d2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
View File

@ -162,6 +162,20 @@ public class UnpackContent extends AbstractProcessor {
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.build();
public static final PropertyDescriptor ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR = new PropertyDescriptor.Builder()
.name("allow-stored-entries-wdd")
.displayName("Allow Stored Entries With Data Descriptor")
.description("Some zip archives contain stored entries with data descriptors which by spec should not " +
"happen. If this property is true they will be read anyway. If false and such an entry is discovered " +
"the zip will fail to process.")
.required(true)
.defaultValue("false")
.sensitive(false)
.allowableValues("true", "false")
.dependsOn(PACKAGING_FORMAT, PackageFormat.ZIP_FORMAT.toString())
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.build();
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Unpacked FlowFiles are sent to this relationship")
@ -195,6 +209,7 @@ public class UnpackContent extends AbstractProcessor {
properties.add(PACKAGING_FORMAT);
properties.add(FILE_FILTER);
properties.add(PASSWORD);
properties.add(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
this.properties = Collections.unmodifiableList(properties);
}
@ -224,7 +239,9 @@ public class UnpackContent extends AbstractProcessor {
if (passwordProperty.isSet()) {
password = passwordProperty.getValue().toCharArray();
}
zipUnpacker = new ZipUnpacker(fileFilter, password);
final PropertyValue allowStoredEntriesWithDataDescriptorVal = context.getProperty(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
final boolean allowStoredEntriesWithDataDescriptor = allowStoredEntriesWithDataDescriptorVal.isSet() ? allowStoredEntriesWithDataDescriptorVal.asBoolean() : false;
zipUnpacker = new ZipUnpacker(fileFilter, password, allowStoredEntriesWithDataDescriptor);
}
}
@ -390,17 +407,19 @@ public class UnpackContent extends AbstractProcessor {
private static class ZipUnpacker extends Unpacker {
private final char[] password;
private final boolean allowStoredEntriesWithDataDescriptor;
public ZipUnpacker(final Pattern fileFilter, final char[] password) {
public ZipUnpacker(final Pattern fileFilter, final char[] password, final boolean allowStoredEntriesWithDataDescriptor) {
super(fileFilter);
this.password = password;
this.allowStoredEntriesWithDataDescriptor = allowStoredEntriesWithDataDescriptor;
}
@Override
public void unpack(final ProcessSession session, final FlowFile source, final List<FlowFile> unpacked) {
final String fragmentId = UUID.randomUUID().toString();
if (password == null) {
session.read(source, new CompressedZipInputStreamCallback(fileFilter, session, source, unpacked, fragmentId));
session.read(source, new CompressedZipInputStreamCallback(fileFilter, session, source, unpacked, fragmentId, allowStoredEntriesWithDataDescriptor));
} else {
session.read(source, new EncryptedZipInputStreamCallback(fileFilter, session, source, unpacked, fragmentId, password));
}
@ -466,19 +485,24 @@ public class UnpackContent extends AbstractProcessor {
}
private static class CompressedZipInputStreamCallback extends ZipInputStreamCallback {
private boolean allowStoredEntriesWithDataDescriptor;
private CompressedZipInputStreamCallback(
final Pattern fileFilter,
final ProcessSession session,
final FlowFile sourceFlowFile,
final List<FlowFile> unpacked,
final String fragmentId
final String fragmentId,
final boolean allowStoredEntriesWithDataDescriptor
) {
super(fileFilter, session, sourceFlowFile, unpacked, fragmentId);
this.allowStoredEntriesWithDataDescriptor = allowStoredEntriesWithDataDescriptor;
}
@Override
public void process(final InputStream inputStream) throws IOException {
try (final ZipArchiveInputStream zipInputStream = new ZipArchiveInputStream(new BufferedInputStream(inputStream))) {
try (final ZipArchiveInputStream zipInputStream = new ZipArchiveInputStream(new BufferedInputStream(inputStream), null, true, allowStoredEntriesWithDataDescriptor)) {
ZipArchiveEntry zipEntry;
while ((zipEntry = zipInputStream.getNextZipEntry()) != null) {
processEntry(zipInputStream, zipEntry.isDirectory(), zipEntry.getName(), EncryptionMethod.NONE);

8
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
View File

@ -155,6 +155,7 @@ public class TestUnpackContent {
final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR, "true"); //just forces this to be exercised
autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
unpackRunner.enqueue(dataPath.resolve("data.zip"));
unpackRunner.enqueue(dataPath.resolve("data.zip"));
@ -192,6 +193,7 @@ public class TestUnpackContent {
final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR, "false");
autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
unpackRunner.enqueue(dataPath.resolve("invalid_data.zip"));
unpackRunner.enqueue(dataPath.resolve("invalid_data.zip"));
@ -253,6 +255,8 @@ public class TestUnpackContent {
final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent());
unpackRunner.setProperty(UnpackContent.FILE_FILTER, "^folder/date.txt$");
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR, "false");
autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
autoUnpackRunner.setProperty(UnpackContent.FILE_FILTER, "^folder/cal.txt$");
unpackRunner.enqueue(dataPath.resolve("data.zip"));
@ -387,6 +391,7 @@ public class TestUnpackContent {
public void testZipThenMerge() throws IOException {
final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR, "false");
unpackRunner.enqueue(dataPath.resolve("data.zip"));
unpackRunner.run();
@ -424,6 +429,7 @@ public class TestUnpackContent {
public void testZipHandlesBadData() throws IOException {
final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR, "false");
unpackRunner.enqueue(dataPath.resolve("data.tar"));
unpackRunner.run();
@ -480,7 +486,7 @@ public class TestUnpackContent {
private void runZipEncryptionMethod(final EncryptionMethod encryptionMethod) throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new UnpackContent());
runner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
runner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR, "false");
final String password = String.class.getSimpleName();
runner.setProperty(UnpackContent.PASSWORD, password);