NIFI-5979 : enhanced ReplaceText processor with "Number of Occurrences" and "Occurrence offset" configurations

Fixed indentation errors to pass checkstyle-checks

Added Evaluation Modes as per discussion in PR thread

Adding exclusions of test files on rat plugin

Added new property 'Line-by-Line Evaluation Mode' and refactored common code

This closes #3375.

Signed-off-by: Koji Kawamura <ijokarumawak@apache.org>
This commit is contained in:
pushpavanthar 2019-03-17 22:19:37 +05:30 committed by Koji Kawamura
parent 25cb29e109
commit d222f14a9e
11 changed files with 687 additions and 166 deletions

View File

@ -429,6 +429,14 @@
<exclude>src/test/resources/TestReplaceTextLineByLine/testFile.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/AppendLineByLineTest.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/PrependLineByLineTest.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/ReplaceLastLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/ReplaceFirstLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/ReplaceExceptLastLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/ReplaceExceptFirstLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/LiteralReplaceLastLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/LiteralReplaceFirstLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/LiteralReplaceExceptLastLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextLineByLine/LiteralReplaceExceptFirstLine.txt</exclude>
<exclude>src/test/resources/TestReplaceTextWithMapping/color-fruit-backreference-mapping.txt</exclude>
<exclude>src/test/resources/TestReplaceTextWithMapping/color-fruit-blank-mapping.txt</exclude>
<exclude>src/test/resources/TestReplaceTextWithMapping/color-fruit-escaped-dollar-mapping.txt</exclude>

View File

@ -53,6 +53,7 @@ import org.apache.nifi.stream.io.StreamUtils;
import org.apache.nifi.stream.io.util.LineDemarcator;
import org.apache.nifi.util.StopWatch;
import javax.annotation.Nullable;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
@ -85,6 +86,11 @@ public class ReplaceText extends AbstractProcessor {
// Constants
public static final String LINE_BY_LINE = "Line-by-Line";
public static final String ALL = "All";
public static final String FIRST_LINE = "First-Line";
public static final String EXCEPT_FIRST_LINE = "Except-First-Line";
public static final String LAST_LINE = "Last-Line";
public static final String EXCEPT_LAST_LINE = "Except-Last-Line";
public static final String ENTIRE_TEXT = "Entire text";
public static final String prependValue = "Prepend";
public static final String appendValue = "Append";
@ -99,10 +105,14 @@ public class ReplaceText extends AbstractProcessor {
// Properties PREPEND, APPEND, REGEX_REPLACE, LITERAL_REPLACE
static final AllowableValue PREPEND = new AllowableValue(prependValue, prependValue,
"Insert the Replacement Value at the beginning of the FlowFile or the beginning of each line (depending on the Evaluation Mode). For \"Line-by-Line\" Evaluation Mode, "
+ "the value will be prepended to each line. For \"Entire Text\" evaluation mode, the value will be prepended to the entire text.");
+ "the value will be prepended to each line. Similarly, for \"First-Line\", \"Last-Line\", \"Except-Last-Line\" and \"Except-First-Line\" Evaluation Modes,"
+ "the value will be prepended to header alone, footer alone, all lines except header and all lines except footer respectively. For \"Entire Text\" evaluation mode,"
+ "the value will be prepended to the entire text.");
static final AllowableValue APPEND = new AllowableValue(appendValue, appendValue,
"Insert the Replacement Value at the end of the FlowFile or the end of each line (depending on the Evaluation Mode). For \"Line-by-Line\" Evaluation Mode, "
+ "the value will be appended to each line. For \"Entire Text\" evaluation mode, the value will be appended to the entire text.");
+ "the value will be appended to each line. Similarly, for \"First-Line\", \"Last-Line\", \"Except-Last-Line\" and \"Except-First-Line\" Evaluation Modes,"
+ "the value will be appended to header alone, footer alone, all lines except header and all lines except footer respectively. For \"Entire Text\" evaluation mode,"
+ "the value will be appended to the entire text.");
static final AllowableValue LITERAL_REPLACE = new AllowableValue(literalReplaceValue, literalReplaceValue,
"Search for all instances of the Search Value and replace the matches with the Replacement Value.");
static final AllowableValue REGEX_REPLACE = new AllowableValue(regexReplaceValue, regexReplaceValue,
@ -161,13 +171,22 @@ public class ReplaceText extends AbstractProcessor {
.build();
public static final PropertyDescriptor EVALUATION_MODE = new PropertyDescriptor.Builder()
.name("Evaluation Mode")
.description("Run the 'Replacement Strategy' against each line separately (Line-by-Line) or buffer the entire file into memory (Entire Text) "
+ "and run against that.")
.description("Run the 'Replacement Strategy' against each line separately (Line-by-Line) or buffer the entire file "
+ "into memory (Entire Text) and run against that.")
.allowableValues(LINE_BY_LINE, ENTIRE_TEXT)
.defaultValue(ENTIRE_TEXT)
.required(true)
.build();
public static final PropertyDescriptor LINE_BY_LINE_EVALUATION_MODE = new PropertyDescriptor.Builder()
.name("Line-by-Line Evaluation Mode")
.description("Run the 'Replacement Strategy' against each line separately (Line-by-Line) for all lines in the FlowFile, First Line (Header) alone, "
+ "Last Line (Footer) alone, Except the First Line (Header) or Except the Last Line (Footer).")
.allowableValues(ALL, FIRST_LINE, LAST_LINE, EXCEPT_FIRST_LINE, EXCEPT_LAST_LINE)
.defaultValue(ALL)
.required(false)
.build();
// Relationships
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
@ -191,6 +210,7 @@ public class ReplaceText extends AbstractProcessor {
properties.add(MAX_BUFFER_SIZE);
properties.add(REPLACEMENT_STRATEGY);
properties.add(EVALUATION_MODE);
properties.add(LINE_BY_LINE_EVALUATION_MODE);
this.properties = Collections.unmodifiableList(properties);
final Set<Relationship> relationships = new HashSet<>();
@ -351,7 +371,7 @@ public class ReplaceText extends AbstractProcessor {
return value;
}
private static class AlwaysReplace implements ReplacementStrategyExecutor {
private class AlwaysReplace implements ReplacementStrategyExecutor {
@Override
public FlowFile replace(FlowFile flowFile, final ProcessSession session, final ProcessContext context, final String evaluateMode, final Charset charset, final int maxBufferSize) {
@ -366,46 +386,40 @@ public class ReplaceText extends AbstractProcessor {
}
});
} else {
flowFile = session.write(flowFile, new StreamCallback() {
@Override
public void process(final InputStream in, final OutputStream out) throws IOException {
try (final LineDemarcator demarcator = new LineDemarcator(in, charset, maxBufferSize, 8192);
final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, charset))) {
String line;
while ((line = demarcator.nextLine()) != null) {
// We need to determine what line ending was used and use that after our replacement value.
lineEndingBuilder.setLength(0);
for (int i = line.length() - 1; i >= 0; i--) {
final char c = line.charAt(i);
if (c == '\r' || c == '\n') {
lineEndingBuilder.append(c);
} else {
break;
}
}
bw.write(replacementValue);
// Preserve original line endings. Reverse string because we iterated over original line ending in reverse order, appending to builder.
// So if builder has multiple characters, they are now reversed from the original string's ordering.
bw.write(lineEndingBuilder.reverse().toString());
}
}
}
});
flowFile = session.write(flowFile, new StreamReplaceCallback(this, charset, maxBufferSize, context, flowFile, null));
}
return flowFile;
}
public void replaceInLine(BufferedWriter bw, String oneLine, @Nullable Matcher matcher, @Nullable Pattern searchPattern, ProcessContext context, FlowFile flowFile) throws IOException {
final String replacementValue = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
final StringBuilder lineEndingBuilder = new StringBuilder(2);
// We need to determine what line ending was used and use that after our replacement value.
lineEndingBuilder.setLength(0);
for (int i = oneLine.length() - 1; i >= 0; i--) {
final char c = oneLine.charAt(i);
if (c == '\r' || c == '\n') {
lineEndingBuilder.append(c);
} else {
break;
}
}
bw.write(replacementValue);
// Preserve original line endings. Reverse string because we iterated over original line ending in reverse order, appending to builder.
// So if builder has multiple characters, they are now reversed from the original string's ordering.
bw.write(lineEndingBuilder.reverse().toString());
}
@Override
public boolean isAllDataBufferedForEntireText() {
return false;
}
}
private static class PrependReplace implements ReplacementStrategyExecutor {
private class PrependReplace implements ReplacementStrategyExecutor {
@Override
public FlowFile replace(FlowFile flowFile, final ProcessSession session, final ProcessContext context, final String evaluateMode, final Charset charset, final int maxBufferSize) {
final String replacementValue = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
@ -419,20 +433,7 @@ public class ReplaceText extends AbstractProcessor {
}
});
} else {
flowFile = session.write(flowFile, new StreamCallback() {
@Override
public void process(final InputStream in, final OutputStream out) throws IOException {
try (final LineDemarcator demarcator = new LineDemarcator(in, charset, maxBufferSize, 8192);
final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, charset))) {
String oneLine;
while (null != (oneLine = demarcator.nextLine())) {
final String updatedValue = replacementValue.concat(oneLine);
bw.write(updatedValue);
}
}
}
});
flowFile = session.write(flowFile, new StreamReplaceCallback(this, charset, maxBufferSize, context, flowFile, null));
}
return flowFile;
}
@ -441,9 +442,15 @@ public class ReplaceText extends AbstractProcessor {
public boolean isAllDataBufferedForEntireText() {
return false;
}
@Override
public void replaceInLine(BufferedWriter bw, String oneLine, @Nullable Matcher matcher, @Nullable Pattern searchPattern, ProcessContext context, FlowFile flowFile) throws IOException {
final String replacementValue = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
bw.write(replacementValue.concat(oneLine));
}
}
private static class AppendReplace implements ReplacementStrategyExecutor {
private class AppendReplace implements ReplacementStrategyExecutor {
@Override
public FlowFile replace(FlowFile flowFile, final ProcessSession session, final ProcessContext context, final String evaluateMode, final Charset charset, final int maxBufferSize) {
@ -458,45 +465,38 @@ public class ReplaceText extends AbstractProcessor {
}
});
} else {
flowFile = session.write(flowFile, new StreamCallback() {
@Override
public void process(final InputStream in, final OutputStream out) throws IOException {
try (final LineDemarcator demarcator = new LineDemarcator(in, charset, maxBufferSize, 8192);
final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, charset))) {
String oneLine;
while (null != (oneLine = demarcator.nextLine())) {
// we need to find the first carriage return or new-line so that we can append the new value
// before the line separate. However, we don't want to do this using a regular expression due
// to performance concerns. So we will find the first occurrence of either \r or \n and use
// that to insert the replacement value.
boolean foundNewLine = false;
for (int i = 0; i < oneLine.length(); i++) {
final char c = oneLine.charAt(i);
if (foundNewLine) {
bw.write(c);
continue;
}
if (c == '\r' || c == '\n') {
bw.write(replacementValue);
foundNewLine = true;
}
bw.write(c);
}
if (!foundNewLine) {
bw.write(replacementValue);
}
}
}
}
});
flowFile = session.write(flowFile, new StreamReplaceCallback(this, charset, maxBufferSize, context, flowFile, null));
}
return flowFile;
}
public void replaceInLine(BufferedWriter bw, String oneLine, @Nullable Matcher matcher, @Nullable Pattern searchPattern, ProcessContext context, FlowFile flowFile) throws IOException {
String replacementValue = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
// we need to find the first carriage return or new-line so that we can append the new value
// before the line separate. However, we don't want to do this using a regular expression due
// to performance concerns. So we will find the first occurrence of either \r or \n and use
// that to insert the replacement value.
boolean foundNewLine = false;
for (int i = 0; i < oneLine.length(); i++) {
final char c = oneLine.charAt(i);
if (foundNewLine) {
bw.write(c);
continue;
}
if (c == '\r' || c == '\n') {
bw.write(replacementValue);
foundNewLine = true;
}
bw.write(c);
}
if (!foundNewLine) {
bw.write(replacementValue);
}
}
@Override
public boolean isAllDataBufferedForEntireText() {
return false;
@ -504,13 +504,13 @@ public class ReplaceText extends AbstractProcessor {
}
private static class RegexReplace implements ReplacementStrategyExecutor {
private class RegexReplace implements ReplacementStrategyExecutor {
private final byte[] buffer;
private final int numCapturingGroups;
private final Map<String, String> additionalAttrs;
// back references are not supported in the evaluated expression
private static final AttributeValueDecorator escapeBackRefDecorator = new AttributeValueDecorator() {
private final AttributeValueDecorator escapeBackRefDecorator = new AttributeValueDecorator() {
@Override
public String decorate(final String attributeValue) {
// when we encounter a '$[0-9+]' replace it with '\$[0-9+]'
@ -580,67 +580,55 @@ public class ReplaceText extends AbstractProcessor {
}
} else {
updatedFlowFile = session.write(flowFile, new StreamCallback() {
@Override
public void process(final InputStream in, final OutputStream out) throws IOException {
try (final LineDemarcator demarcator = new LineDemarcator(in, charset, maxBufferSize, 8192);
final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, charset))) {
String oneLine;
final StringBuffer sb = new StringBuffer();
Matcher matcher = null;
while (null != (oneLine = demarcator.nextLine())) {
additionalAttrs.clear();
if (matcher == null) {
matcher = searchPattern.matcher(oneLine);
} else {
matcher.reset(oneLine);
}
int matches = 0;
sb.setLength(0);
while (matcher.find()) {
matches++;
for (int i=0; i <= matcher.groupCount(); i++) {
additionalAttrs.put("$" + i, matcher.group(i));
}
String replacement = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile, additionalAttrs, escapeBackRefDecorator).getValue();
replacement = escapeLiteralBackReferences(replacement, numCapturingGroups);
String replacementFinal = normalizeReplacementString(replacement);
matcher.appendReplacement(sb, replacementFinal);
}
if (matches > 0) {
matcher.appendTail(sb);
final String updatedValue = sb.toString();
bw.write(updatedValue);
} else {
// No match. Just write out the line as it was.
bw.write(oneLine);
}
}
}
}
});
updatedFlowFile = session.write(flowFile, new StreamReplaceCallback(this, charset, maxBufferSize, context, flowFile, searchPattern));
}
return updatedFlowFile;
}
public void replaceInLine(BufferedWriter bw, String oneLine, @Nullable Matcher matcher, Pattern searchPattern, ProcessContext context, FlowFile flowFile) throws IOException {
additionalAttrs.clear();
if (matcher == null) {
matcher = searchPattern.matcher(oneLine);
} else {
matcher.reset(oneLine);
}
int matches = 0;
StringBuffer sb = new StringBuffer();
while (matcher.find()) {
matches++;
for (int i=0; i <= matcher.groupCount(); i++) {
additionalAttrs.put("$" + i, matcher.group(i));
}
String replacement = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile, additionalAttrs, escapeBackRefDecorator).getValue();
replacement = escapeLiteralBackReferences(replacement, numCapturingGroups);
String replacementFinal = normalizeReplacementString(replacement);
matcher.appendReplacement(sb, replacementFinal);
}
if (matches > 0) {
matcher.appendTail(sb);
final String updatedValue = sb.toString();
bw.write(updatedValue);
} else {
// No match. Just write out the line as it was.
bw.write(oneLine);
}
}
@Override
public boolean isAllDataBufferedForEntireText() {
return true;
}
}
private static class LiteralReplace implements ReplacementStrategyExecutor {
private class LiteralReplace implements ReplacementStrategyExecutor {
private final byte[] buffer;
public LiteralReplace(final byte[] buffer) {
@ -667,42 +655,34 @@ public class ReplaceText extends AbstractProcessor {
}
});
} else {
final int initialBufferSize = (int) Math.min(flowFile.getSize(), 8192);
final Pattern searchPattern = Pattern.compile(searchValue, Pattern.LITERAL);
flowFile = session.write(flowFile, new StreamCallback() {
@Override
public void process(final InputStream in, final OutputStream out) throws IOException {
try (final LineDemarcator demarcator = new LineDemarcator(in, charset, maxBufferSize, initialBufferSize);
final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, charset))) {
String oneLine;
while (null != (oneLine = demarcator.nextLine())) {
int matches = 0;
int lastEnd = 0;
final Matcher matcher = searchPattern.matcher(oneLine);
while (matcher.find()) {
bw.write(oneLine, lastEnd, matcher.start() - lastEnd);
bw.write(replacementValue);
matches++;
lastEnd = matcher.end();
}
if (matches > 0) {
bw.write(oneLine, lastEnd, oneLine.length() - lastEnd);
} else {
bw.write(oneLine);
}
}
}
}
});
flowFile = session.write(flowFile, new StreamReplaceCallback(this, charset, maxBufferSize, context, flowFile, searchPattern));
}
return flowFile;
}
public void replaceInLine(BufferedWriter bw, String oneLine, @Nullable Matcher matcher, @Nullable Pattern searchPattern, ProcessContext context, FlowFile flowFile) throws IOException {
String replacementValue = context.getProperty(REPLACEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
int matches = 0;
int lastEnd = 0;
while (matcher.find()) {
bw.write(oneLine, lastEnd, matcher.start() - lastEnd);
bw.write(replacementValue);
matches++;
lastEnd = matcher.end();
}
if (matches > 0) {
bw.write(oneLine, lastEnd, oneLine.length() - lastEnd);
} else {
bw.write(oneLine);
}
}
@Override
public boolean isAllDataBufferedForEntireText() {
return true;
@ -726,5 +706,76 @@ public class ReplaceText extends AbstractProcessor {
FlowFile replace(FlowFile flowFile, ProcessSession session, ProcessContext context, String evaluateMode, Charset charset, int maxBufferSize);
boolean isAllDataBufferedForEntireText();
void replaceInLine(BufferedWriter bw, String oneLine, @Nullable Matcher matcher, @Nullable Pattern searchPattern, ProcessContext context, FlowFile flowFile) throws IOException ;
}
private class StreamReplaceCallback implements StreamCallback {
private final Charset charset;
private final int maxBufferSize;
private final ProcessContext context;
private final FlowFile flowFile;
private final ReplacementStrategyExecutor replacementStrategyExecutor;
private final Pattern searchPattern;
public StreamReplaceCallback(ReplacementStrategyExecutor replacementStrategyExecutor,
Charset charset,
int maxBufferSize,
ProcessContext context,
FlowFile flowFile,
@Nullable Pattern searchPattern) {
this.replacementStrategyExecutor = replacementStrategyExecutor;
this.charset = charset;
this.maxBufferSize = maxBufferSize;
this.context = context;
this.flowFile = flowFile;
this.searchPattern = searchPattern;
}
@Override
public void process(final InputStream in, final OutputStream out) throws IOException {
final String lineByLineEvaluationMode = context.getProperty(LINE_BY_LINE_EVALUATION_MODE).getValue();
try (final LineDemarcator demarcator = new LineDemarcator(in, charset, maxBufferSize, 8192);
final BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, charset))) {
String precedingLine = demarcator.nextLine();
String succeedingLine;
Matcher matcher = null;
boolean firstLine = true;
while (null != (succeedingLine = demarcator.nextLine())) {
matcher = null != searchPattern ? searchPattern.matcher(precedingLine) : null;
if(firstLine && lineByLineEvaluationMode.equalsIgnoreCase(FIRST_LINE)){
replacementStrategyExecutor.replaceInLine(bw, precedingLine, matcher, searchPattern, context, flowFile);
firstLine = false;
} else if(firstLine && lineByLineEvaluationMode.equalsIgnoreCase(EXCEPT_FIRST_LINE)) {
firstLine = false;
bw.write(precedingLine);
} else if(lineByLineEvaluationMode.equalsIgnoreCase(LINE_BY_LINE)
|| lineByLineEvaluationMode.equalsIgnoreCase(EXCEPT_LAST_LINE)
|| lineByLineEvaluationMode.equalsIgnoreCase(ALL)
|| (!firstLine && lineByLineEvaluationMode.equalsIgnoreCase(EXCEPT_FIRST_LINE))) {
replacementStrategyExecutor.replaceInLine(bw, precedingLine, matcher, searchPattern, context, flowFile);
} else {
bw.write(precedingLine);
}
precedingLine = succeedingLine;
}
// 0 byte empty FlowFIles are left untouched
if(null != precedingLine) {
if (lineByLineEvaluationMode.equalsIgnoreCase(EXCEPT_LAST_LINE)
|| (!firstLine && lineByLineEvaluationMode.equalsIgnoreCase(FIRST_LINE))
|| (firstLine && lineByLineEvaluationMode.equalsIgnoreCase(EXCEPT_FIRST_LINE))) {
bw.write(precedingLine);
} else {
matcher = null != searchPattern ? searchPattern.matcher(precedingLine) : null;
replacementStrategyExecutor.replaceInLine(bw, precedingLine, matcher, searchPattern, context, flowFile);
}
}
}
}
}
}

View File

@ -166,6 +166,71 @@ public class TestReplaceText {
out.assertContentEquals("_hello\n_there\n_madam".getBytes("UTF-8"));
}
@Test
public void testPrependFirstLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "_");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.PREPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.FIRST_LINE);
runner.enqueue("hello\nthere\nmadam".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("_hello\nthere\nmadam".getBytes("UTF-8"));
}
@Test
public void testPrependLastLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "_");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.PREPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.LAST_LINE);
runner.enqueue("hello\nthere\nmadam".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello\nthere\n_madam".getBytes("UTF-8"));
}
@Test
public void testPrependExceptFirstLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "_");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.PREPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_FIRST_LINE);
runner.enqueue("hello\nthere\nmadam".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello\n_there\n_madam".getBytes("UTF-8"));
}
@Test
public void testPrependExceptLastLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "_");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.PREPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_LAST_LINE);
runner.enqueue("hello\nthere\nmadam".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("_hello\n_there\nmadam".getBytes("UTF-8"));
}
@Test
public void testAppendSimple() throws IOException {
final TestRunner runner = getRunner();
@ -195,6 +260,71 @@ public class TestReplaceText {
out.assertContentEquals("hello!\rthere!\rsir!");
}
@Test
public void testAppendFirstLineWithCarriageReturn() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.FIRST_LINE);
runner.enqueue("hello\rthere\rsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello!\rthere\rsir");
}
@Test
public void testAppendExceptFirstLineWithCarriageReturn() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_FIRST_LINE);
runner.enqueue("hello\rthere\rsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello\rthere!\rsir!");
}
@Test
public void testAppendLastLineWithCarriageReturn() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.LAST_LINE);
runner.enqueue("hello\rthere\rsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello\rthere\rsir!");
}
@Test
public void testAppendExceptLastLineWithCarriageReturn() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_LAST_LINE);
runner.enqueue("hello\rthere\rsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello!\rthere!\rsir");
}
@Test
public void testAppendWithNewLine() throws IOException {
final TestRunner runner = getRunner();
@ -225,6 +355,74 @@ public class TestReplaceText {
out.assertContentEquals("hello!\r\nthere!\r\nsir!");
}
@Test
public void testAppendFirstLineWithCarriageReturnNewLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.FIRST_LINE);
runner.enqueue("hello\r\nthere\r\nsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello!\r\nthere\r\nsir");
}
@Test
public void testAppendLastLineWithCarriageReturnNewLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.LAST_LINE);
runner.enqueue("hello\r\nthere\r\nsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello\r\nthere\r\nsir!");
}
@Test
public void testAppendExceptFistLineWithCarriageReturnNewLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_FIRST_LINE);
runner.enqueue("hello\r\nthere\r\nsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello\r\nthere!\r\nsir!");
}
@Test
public void testAppendExceptLastLineWithCarriageReturnNewLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_LAST_LINE);
runner.enqueue("hello\r\nthere\r\nsir".getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals("hello!\r\nthere!\r\nsir");
}
@Test
public void testLiteralSimple() throws IOException {
final TestRunner runner = getRunner();
@ -648,6 +846,24 @@ public class TestReplaceText {
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/food.txt")));
}
@Test
public void testZeroByteContentFileLineByLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "odo");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "ood");
final File zeroByteFile = File.createTempFile("zeroByte", ".txt");
runner.enqueue(translateNewLines(zeroByteFile.getPath()));
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(zeroByteFile.getPath()));
}
@Test
public void testPrependSimpleLineByLine() throws IOException {
final TestRunner runner = getRunner();
@ -847,6 +1063,164 @@ public class TestReplaceText {
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/Blu$2e_clu$2e.txt")));
}
@Test
public void testBackReferenceWithTooLargeOfIndexIsEscapedFirstLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.FIRST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "(H)");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1$2");
final Map<String, String> attributes = new HashMap<>();
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")), attributes);
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/ReplaceFirstLine.txt")));
}
@Test
public void testBackReferenceWithTooLargeOfIndexIsEscapedLastLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.LAST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "(O)");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1$2");
final Map<String, String> attributes = new HashMap<>();
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")), attributes);
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/ReplaceLastLine.txt")));
}
@Test
public void testBackReferenceWithTooLargeOfIndexIsEscapedExceptFirstLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_FIRST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "(H)");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1$2");
final Map<String, String> attributes = new HashMap<>();
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")), attributes);
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/ReplaceExceptFirstLine.txt")));
}
@Test
public void testBackReferenceWithTooLargeOfIndexIsEscapedExceptLastLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_LAST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "(O)");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1$2");
final Map<String, String> attributes = new HashMap<>();
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")), attributes);
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/ReplaceExceptLastLine.txt")));
}
@Test
public void testLiteralBackReferenceFistLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.FIRST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "H");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "[$1]");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.LITERAL_REPLACE);
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")));
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/LiteralReplaceFirstLine.txt")));
}
@Test
public void testLiteralBackReferenceExceptFirstLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_FIRST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "H");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "[$1]");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.LITERAL_REPLACE);
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")));
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/LiteralReplaceExceptFirstLine.txt")));
}
@Test
public void testLiteralBackReferenceLastLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.LAST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "O");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "[$1]");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.LITERAL_REPLACE);
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")));
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/LiteralReplaceLastLine.txt")));
}
@Test
public void testLiteralBackReferenceExceptLastLine() throws IOException {
final TestRunner runner = getRunner();
runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
runner.setProperty(ReplaceText.LINE_BY_LINE_EVALUATION_MODE, ReplaceText.EXCEPT_LAST_LINE);
runner.setProperty(ReplaceText.SEARCH_VALUE, "O");
runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "[$1]");
runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.LITERAL_REPLACE);
runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")));
runner.run();
runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/LiteralReplaceExceptLastLine.txt")));
}
@Test
public void testBackReferenceWithInvalidReferenceIsEscapedLineByLine() throws IOException {
final TestRunner runner = getRunner();

View File

@ -0,0 +1,11 @@
<<<HEADER>>>
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley [$1]uey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley [$1]uey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley [$1]uey
<<<FOOTER>>>

View File

@ -0,0 +1,11 @@
<<<HEADER>>>
Fodo D[$1]D[$1] cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo D[$1]D[$1] cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo D[$1]D[$1] cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
<<<FOOTER>>>

View File

@ -0,0 +1,11 @@
<<<[$1]EADER>>>
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
<<<FOOTER>>>

View File

@ -0,0 +1,11 @@
<<<HEADER>>>
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
<<<F[$1][$1]TER>>>

View File

@ -0,0 +1,11 @@
<<<HEADER>>>
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley H$2uey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley H$2uey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley H$2uey
<<<FOOTER>>>

View File

@ -0,0 +1,11 @@
<<<HEADER>>>
Fodo DO$2DO$2 cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DO$2DO$2 cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DO$2DO$2 cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
<<<FOOTER>>>

View File

@ -0,0 +1,11 @@
<<<H$2EADER>>>
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
<<<FOOTER>>>

View File

@ -0,0 +1,11 @@
<<<HEADER>>>
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
Fodo DODO cujo Pojo
Blue Dew clue hew
Grampa Riley Huey
<<<FO$2O$2TER>>>