Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-4562: This closes #2331. If an Exception is thrown when merging FlowFiles, ensure that we remove the 'bundle' flowfile before exiting the method 

Signed-off-by: joewitt <joewitt@apache.org>
This commit is contained in:
Mark Payne 2017-12-08 13:09:00 -05:00 committed by joewitt
parent 3b74d2ddad
commit 1fc1d38fd8
1 changed files with 215 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
View File

@ -578,6 +578,7 @@ public class MergeContent extends BinFiles {
final ProcessSession session = bin.getSession(); final ProcessSession session = bin.getSession();
FlowFile bundle = session.create(bin.getContents()); FlowFile bundle = session.create(bin.getContents());
final AtomicReference<String> bundleMimeTypeRef = new AtomicReference<>(null); final AtomicReference<String> bundleMimeTypeRef = new AtomicReference<>(null);
try {
bundle = session.write(bundle, new OutputStreamCallback() { bundle = session.write(bundle, new OutputStreamCallback() {
@Override @Override
public void process(final OutputStream out) throws IOException { public void process(final OutputStream out) throws IOException {
@ -621,6 +622,10 @@ public class MergeContent extends BinFiles {
} }
} }
}); });
} catch (final Exception e) {
session.remove(bundle);
throw e;
}
session.getProvenanceReporter().join(contents, bundle); session.getProvenanceReporter().join(contents, bundle);
bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents)); bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents));
@ -719,6 +724,7 @@ public class MergeContent extends BinFiles {
final boolean keepPath = context.getProperty(KEEP_PATH).asBoolean(); final boolean keepPath = context.getProperty(KEEP_PATH).asBoolean();
FlowFile bundle = session.create(); // we don't pass the parents to the #create method because the parents belong to different sessions FlowFile bundle = session.create(); // we don't pass the parents to the #create method because the parents belong to different sessions
try {
bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents) + ".tar"); bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents) + ".tar");
bundle = session.write(bundle, new OutputStreamCallback() { bundle = session.write(bundle, new OutputStreamCallback() {
@Override @Override
@ -738,7 +744,7 @@ public class MergeContent extends BinFiles {
tarEntry.setMode(Integer.parseInt(permissionsVal)); tarEntry.setMode(Integer.parseInt(permissionsVal));
} catch (final Exception e) { } catch (final Exception e) {
getLogger().debug("Attribute {} of {} is set to {}; expected 3 digits between 0-7, so ignoring", getLogger().debug("Attribute {} of {} is set to {}; expected 3 digits between 0-7, so ignoring",
new Object[]{TAR_PERMISSIONS_ATTRIBUTE, flowFile, permissionsVal}); new Object[] {TAR_PERMISSIONS_ATTRIBUTE, flowFile, permissionsVal});
} }
} }
@ -749,7 +755,7 @@ public class MergeContent extends BinFiles {
tarEntry.setModTime(Instant.parse(modTime).toEpochMilli()); tarEntry.setModTime(Instant.parse(modTime).toEpochMilli());
} catch (final Exception e) { } catch (final Exception e) {
getLogger().debug("Attribute {} of {} is set to {}; expected ISO8601 format, so ignoring", getLogger().debug("Attribute {} of {} is set to {}; expected ISO8601 format, so ignoring",
new Object[]{TAR_MODIFIED_TIME, flowFile, modTime}); new Object[] {TAR_MODIFIED_TIME, flowFile, modTime});
} }
} }
@ -761,6 +767,10 @@ public class MergeContent extends BinFiles {
} }
} }
}); });
} catch (final Exception e) {
session.remove(bundle);
throw e;
}
bin.getSession().getProvenanceReporter().join(contents, bundle); bin.getSession().getProvenanceReporter().join(contents, bundle);
return bundle; return bundle;
@ -794,6 +804,7 @@ public class MergeContent extends BinFiles {
FlowFile bundle = session.create(contents); FlowFile bundle = session.create(contents);
try {
bundle = session.write(bundle, new OutputStreamCallback() { bundle = session.write(bundle, new OutputStreamCallback() {
@Override @Override
public void process(final OutputStream rawOut) throws IOException { public void process(final OutputStream rawOut) throws IOException {
@ -823,6 +834,10 @@ public class MergeContent extends BinFiles {
} }
} }
}); });
} catch (final Exception e) {
session.remove(bundle);
throw e;
}
bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents) + ".pkg"); bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents) + ".pkg");
session.getProvenanceReporter().join(contents, bundle); session.getProvenanceReporter().join(contents, bundle);
@ -860,6 +875,7 @@ public class MergeContent extends BinFiles {
FlowFile bundle = session.create(contents); FlowFile bundle = session.create(contents);
try {
bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents) + ".zip"); bundle = session.putAttribute(bundle, CoreAttributes.FILENAME.key(), createFilename(contents) + ".zip");
bundle = session.write(bundle, new OutputStreamCallback() { bundle = session.write(bundle, new OutputStreamCallback() {
@Override @Override
@ -879,7 +895,7 @@ public class MergeContent extends BinFiles {
out.closeEntry(); out.closeEntry();
unmerged.remove(flowFile); unmerged.remove(flowFile);
} catch (ZipException e) { } catch (ZipException e) {
getLogger().error("Encountered exception merging {}", new Object[]{flowFile}, e); getLogger().error("Encountered exception merging {}", new Object[] {flowFile}, e);
} }
} }
@ -888,6 +904,10 @@ public class MergeContent extends BinFiles {
} }
} }
}); });
} catch (final Exception e) {
session.remove(bundle);
throw e;
}
session.getProvenanceReporter().join(contents, bundle); session.getProvenanceReporter().join(contents, bundle);
return bundle; return bundle;
@ -921,6 +941,7 @@ public class MergeContent extends BinFiles {
// we don't pass the parents to the #create method because the parents belong to different sessions // we don't pass the parents to the #create method because the parents belong to different sessions
FlowFile bundle = session.create(contents); FlowFile bundle = session.create(contents);
try {
bundle = session.write(bundle, new OutputStreamCallback() { bundle = session.write(bundle, new OutputStreamCallback() {
@Override @Override
public void process(final OutputStream rawOut) throws IOException { public void process(final OutputStream rawOut) throws IOException {
@ -955,7 +976,7 @@ public class MergeContent extends BinFiles {
// check that we're appending to the same schema // check that we're appending to the same schema
if (!schema.get().equals(reader.getSchema())) { if (!schema.get().equals(reader.getSchema())) {
getLogger().debug("Input file {} has different schema - {}, not merging", getLogger().debug("Input file {} has different schema - {}, not merging",
new Object[]{flowFile.getId(), reader.getSchema().getName()}); new Object[] {flowFile.getId(), reader.getSchema().getName()});
canMerge = false; canMerge = false;
unmerged.add(flowFile); unmerged.add(flowFile);
} }
@ -971,7 +992,7 @@ public class MergeContent extends BinFiles {
// Ignore additional metadata if ALL_COMMON is the strategy, otherwise don't merge // Ignore additional metadata if ALL_COMMON is the strategy, otherwise don't merge
if (!METADATA_STRATEGY_ALL_COMMON.getValue().equals(metadataStrategy) || writersMetadatum != null) { if (!METADATA_STRATEGY_ALL_COMMON.getValue().equals(metadataStrategy) || writersMetadatum != null) {
getLogger().debug("Input file {} has different non-reserved metadata, not merging", getLogger().debug("Input file {} has different non-reserved metadata, not merging",
new Object[]{flowFile.getId()}); new Object[] {flowFile.getId()});
canMerge = false; canMerge = false;
unmerged.add(flowFile); unmerged.add(flowFile);
} }
@ -987,7 +1008,7 @@ public class MergeContent extends BinFiles {
} }
if (!inputCodec.get().equals(thisCodec)) { if (!inputCodec.get().equals(thisCodec)) {
getLogger().debug("Input file {} has different codec, not merging", getLogger().debug("Input file {} has different codec, not merging",
new Object[]{flowFile.getId()}); new Object[] {flowFile.getId()});
canMerge = false; canMerge = false;
unmerged.add(flowFile); unmerged.add(flowFile);
} }
@ -1007,6 +1028,10 @@ public class MergeContent extends BinFiles {
} }
} }
}); });
} catch (final Exception e) {
session.remove(bundle);
throw e;
}
final Collection<FlowFile> parents; final Collection<FlowFile> parents;
if (unmerged.isEmpty()) { if (unmerged.isEmpty()) {