NIFI-10865 allow RecordPath's unescapeJson to convert de-serialised JSON Objects into Records

NIFI-10865 allow UpdateRecord to replace the Record root for relative paths, e.g. when a RecordPath function is used to modify selected field(s)

This closes #6708

Signed-off-by: Mike Thomsen <mthomsen@apache.org>
This commit is contained in:
Chris Sampson 2022-11-22 11:44:53 +00:00 committed by Mike Thomsen
parent d399c5267f
commit e2067c4ea1
13 changed files with 316 additions and 43 deletions

View File

@ -23,7 +23,10 @@ import org.apache.nifi.record.path.RecordPathEvaluationContext;
import org.apache.nifi.record.path.StandardFieldValue; import org.apache.nifi.record.path.StandardFieldValue;
import org.apache.nifi.record.path.exception.RecordPathException; import org.apache.nifi.record.path.exception.RecordPathException;
import org.apache.nifi.record.path.paths.RecordPathSegment; import org.apache.nifi.record.path.paths.RecordPathSegment;
import org.apache.nifi.record.path.util.RecordPathUtils;
import org.apache.nifi.serialization.record.DataType; import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.type.ArrayDataType; import org.apache.nifi.serialization.record.type.ArrayDataType;
import org.apache.nifi.serialization.record.type.ChoiceDataType; import org.apache.nifi.serialization.record.type.ChoiceDataType;
import org.apache.nifi.serialization.record.type.RecordDataType; import org.apache.nifi.serialization.record.type.RecordDataType;
@ -31,21 +34,29 @@ import org.apache.nifi.serialization.record.util.DataTypeUtils;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
public class UnescapeJson extends RecordPathSegment { public class UnescapeJson extends RecordPathSegment {
private final RecordPathSegment recordPath; private final RecordPathSegment recordPath;
private final RecordPathSegment convertToRecordRecordPath;
private final ObjectMapper objectMapper = new ObjectMapper(); private final ObjectMapper objectMapper = new ObjectMapper();
public UnescapeJson(final RecordPathSegment recordPath, final boolean absolute) { public UnescapeJson(final RecordPathSegment recordPath, final RecordPathSegment convertToRecordRecordPath, final boolean absolute) {
super("unescapeJson", null, absolute); super("unescapeJson", null, absolute);
this.recordPath = recordPath; this.recordPath = recordPath;
this.convertToRecordRecordPath = convertToRecordRecordPath;
} }
@Override @Override
public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
final boolean convertMapToRecord = convertToRecordRecordPath != null
&& Boolean.parseBoolean(RecordPathUtils.getFirstStringValue(convertToRecordRecordPath, context));
final Stream<FieldValue> fieldValues = recordPath.evaluate(context); final Stream<FieldValue> fieldValues = recordPath.evaluate(context);
return fieldValues.filter(fv -> fv.getValue() != null) return fieldValues.filter(fv -> fv.getValue() != null)
.map(fv -> { .map(fv -> {
@ -58,7 +69,10 @@ public class UnescapeJson extends RecordPathSegment {
dataType = DataTypeUtils.chooseDataType(value, (ChoiceDataType) fv.getField().getDataType()); dataType = DataTypeUtils.chooseDataType(value, (ChoiceDataType) fv.getField().getDataType());
} }
return new StandardFieldValue(convertFieldValue(value, fv.getField().getFieldName(), dataType), fv.getField(), fv.getParent().orElse(null)); return new StandardFieldValue(
convertFieldValue(value, fv.getField().getFieldName(), dataType, convertMapToRecord),
fv.getField(), fv.getParent().orElse(null)
);
} catch (IOException e) { } catch (IOException e) {
throw new RecordPathException("Unable to deserialise JSON String into Record Path value", e); throw new RecordPathException("Unable to deserialise JSON String into Record Path value", e);
} }
@ -69,7 +83,7 @@ public class UnescapeJson extends RecordPathSegment {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private Object convertFieldValue(final Object value, final String fieldName, final DataType dataType) throws IOException { private Object convertFieldValue(final Object value, final String fieldName, final DataType dataType, final boolean convertMapToRecord) throws IOException {
if (dataType instanceof RecordDataType) { if (dataType instanceof RecordDataType) {
// convert Maps to Records // convert Maps to Records
final Map<String, Object> map = objectMapper.readValue(value.toString(), Map.class); final Map<String, Object> map = objectMapper.readValue(value.toString(), Map.class);
@ -85,7 +99,20 @@ public class UnescapeJson extends RecordPathSegment {
return arr; return arr;
} else { } else {
// generic conversion for simpler fields // generic conversion for simpler fields
return objectMapper.readValue(value.toString(), Object.class); final Object parsed = objectMapper.readValue(value.toString(), Object.class);
if (convertMapToRecord) {
if (DataTypeUtils.isCompatibleDataType(parsed, RecordFieldType.RECORD.getDataType())) {
return DataTypeUtils.toRecord(parsed, fieldName);
} else if (DataTypeUtils.isArrayTypeCompatible(parsed, RecordFieldType.RECORD.getDataType())) {
return Arrays.stream((Object[]) parsed).map(m -> DataTypeUtils.toRecord(m, fieldName)).toArray(Record[]::new);
} else if (parsed instanceof Collection
&& !((Collection<Object>) parsed).isEmpty()
&& DataTypeUtils.isCompatibleDataType(((Collection<Object>) parsed).stream().findFirst().get(), RecordFieldType.RECORD.getDataType())) {
return ((Collection<Object>) parsed).stream().map(m -> DataTypeUtils.toRecord(m, fieldName)).collect(Collectors.toList());
}
}
return parsed;
} }
} }
} }

View File

@ -329,8 +329,15 @@ public class RecordPathCompiler {
return new EscapeJson(args[0], absolute); return new EscapeJson(args[0], absolute);
} }
case "unescapeJson": { case "unescapeJson": {
final int numArgs = argumentListTree.getChildCount();
if (numArgs == 1) {
final RecordPathSegment[] args = getArgPaths(argumentListTree, 1, functionName, absolute); final RecordPathSegment[] args = getArgPaths(argumentListTree, 1, functionName, absolute);
return new UnescapeJson(args[0], absolute); return new UnescapeJson(args[0], null, absolute);
} else {
final RecordPathSegment[] args = getArgPaths(argumentListTree, 2, functionName, absolute);
return new UnescapeJson(args[0], args[1], absolute);
}
} }
case "hash":{ case "hash":{
final RecordPathSegment[] args = getArgPaths(argumentListTree, 2, functionName, absolute); final RecordPathSegment[] args = getArgPaths(argumentListTree, 2, functionName, absolute);

View File

@ -60,6 +60,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
@SuppressWarnings("OptionalGetWithoutIsPresent")
public class TestRecordPath { public class TestRecordPath {
private static final String USER_TIMEZONE_PROPERTY = "user.timezone"; private static final String USER_TIMEZONE_PROPERTY = "user.timezone";
@ -280,7 +281,7 @@ public class TestRecordPath {
final Record record = new MapRecord(schema, values); final Record record = new MapRecord(schema, values);
final FieldValue fieldValue = RecordPath.compile("/attributes['city']").evaluate(record).getSelectedFields().findFirst().get(); final FieldValue fieldValue = RecordPath.compile("/attributes['city']").evaluate(record).getSelectedFields().findFirst().get();
assertTrue(fieldValue.getField().getFieldName().equals("attributes")); assertEquals("attributes", fieldValue.getField().getFieldName());
assertEquals("New York", fieldValue.getValue()); assertEquals("New York", fieldValue.getValue());
assertEquals(record, fieldValue.getParentRecord().get()); assertEquals(record, fieldValue.getParentRecord().get());
} }
@ -300,7 +301,7 @@ public class TestRecordPath {
final Record record = new MapRecord(schema, values); final Record record = new MapRecord(schema, values);
final FieldValue fieldValue = RecordPath.compile("/attributes/.['city']").evaluate(record).getSelectedFields().findFirst().get(); final FieldValue fieldValue = RecordPath.compile("/attributes/.['city']").evaluate(record).getSelectedFields().findFirst().get();
assertTrue(fieldValue.getField().getFieldName().equals("attributes")); assertEquals("attributes", fieldValue.getField().getFieldName());
assertEquals("New York", fieldValue.getValue()); assertEquals("New York", fieldValue.getValue());
assertEquals(record, fieldValue.getParentRecord().get()); assertEquals(record, fieldValue.getParentRecord().get());
} }
@ -1094,21 +1095,24 @@ public class TestRecordPath {
// Special character cases // Special character cases
values.put("name", "John Doe"); values.put("name", "John Doe");
assertEquals( assertEquals("John\nDoe", RecordPath.compile("replaceRegex(/name, '[\\s]', '\\n')")
"John\nDoe", RecordPath.compile("replaceRegex(/name, '[\\s]', '\\n')") .evaluate(record).getSelectedFields().findFirst().get().getValue(),
.evaluate(record).getSelectedFields().findFirst().get().getValue()); "Replacing whitespace to new line");
values.put("name", "John\nDoe"); values.put("name", "John\nDoe");
assertEquals("John Doe", RecordPath.compile("replaceRegex(/name, '\\n', ' ')") assertEquals("John Doe", RecordPath.compile("replaceRegex(/name, '\\n', ' ')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing new line to whitespace");
values.put("name", "John Doe"); values.put("name", "John Doe");
assertEquals("John\tDoe", RecordPath.compile("replaceRegex(/name, '[\\s]', '\\t')") assertEquals("John\tDoe", RecordPath.compile("replaceRegex(/name, '[\\s]', '\\t')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing whitespace to tab");
values.put("name", "John\tDoe"); values.put("name", "John\tDoe");
assertEquals("John Doe", RecordPath.compile("replaceRegex(/name, '\\t', ' ')") assertEquals("John Doe", RecordPath.compile("replaceRegex(/name, '\\t', ' ')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing tab to whitespace");
} }
@ -1126,23 +1130,27 @@ public class TestRecordPath {
final Record record = new MapRecord(schema, values); final Record record = new MapRecord(schema, values);
// Quotes // Quotes
// NOTE: At Java code, a single back-slash needs to be escaped with another-back slash, but needn't to do so at NiFi UI. // NOTE: At Java code, a single back-slash needs to be escaped with another-back slash, but needn't do so at NiFi UI.
// The test record path is equivalent to replaceRegex(/name, '\'', '"') // The test record path is equivalent to replaceRegex(/name, '\'', '"')
values.put("name", "'John' 'Doe'"); values.put("name", "'John' 'Doe'");
assertEquals("\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name, '\\'', '\"')") assertEquals("\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name, '\\'', '\"')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing quote to double-quote");
values.put("name", "\"John\" \"Doe\""); values.put("name", "\"John\" \"Doe\"");
assertEquals("'John' 'Doe'", RecordPath.compile("replaceRegex(/name, '\"', '\\'')") assertEquals("'John' 'Doe'", RecordPath.compile("replaceRegex(/name, '\"', '\\'')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing double-quote to single-quote");
values.put("name", "'John' 'Doe'"); values.put("name", "'John' 'Doe'");
assertEquals("\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name, \"'\", \"\\\"\")") assertEquals("\"John\" \"Doe\"", RecordPath.compile("replaceRegex(/name, \"'\", \"\\\"\")")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing quote to double-quote, the function arguments are wrapped by double-quote");
values.put("name", "\"John\" \"Doe\""); values.put("name", "\"John\" \"Doe\"");
assertEquals("'John' 'Doe'", RecordPath.compile("replaceRegex(/name, \"\\\"\", \"'\")") assertEquals("'John' 'Doe'", RecordPath.compile("replaceRegex(/name, \"\\\"\", \"'\")")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing double-quote to single-quote, the function arguments are wrapped by double-quote");
} }
@ -1160,15 +1168,17 @@ public class TestRecordPath {
final Record record = new MapRecord(schema, values); final Record record = new MapRecord(schema, values);
// Back-slash // Back-slash
// NOTE: At Java code, a single back-slash needs to be escaped with another-back slash, but needn't to do so at NiFi UI. // NOTE: At Java code, a single back-slash needs to be escaped with another-back slash, but needn't do so at NiFi UI.
// The test record path is equivalent to replaceRegex(/name, '\\', '/') // The test record path is equivalent to replaceRegex(/name, '\\', '/')
values.put("name", "John\\Doe"); values.put("name", "John\\Doe");
assertEquals("John/Doe", RecordPath.compile("replaceRegex(/name, '\\\\', '/')") assertEquals("John/Doe", RecordPath.compile("replaceRegex(/name, '\\\\', '/')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing a back-slash to forward-slash");
values.put("name", "John/Doe"); values.put("name", "John/Doe");
assertEquals("John\\Doe", RecordPath.compile("replaceRegex(/name, '/', '\\\\')") assertEquals("John\\Doe", RecordPath.compile("replaceRegex(/name, '/', '\\\\')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Replacing a forward-slash to back-slash");
} }
@ -1188,11 +1198,13 @@ public class TestRecordPath {
// Brackets // Brackets
values.put("name", "J[o]hn Do[e]"); values.put("name", "J[o]hn Do[e]");
assertEquals("J(o)hn Do(e)", RecordPath.compile("replaceRegex(replaceRegex(/name, '\\[', '('), '\\]', ')')") assertEquals("J(o)hn Do(e)", RecordPath.compile("replaceRegex(replaceRegex(/name, '\\[', '('), '\\]', ')')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Square brackets can be escaped with back-slash");
values.put("name", "J(o)hn Do(e)"); values.put("name", "J(o)hn Do(e)");
assertEquals("J[o]hn Do[e]", RecordPath.compile("replaceRegex(replaceRegex(/name, '\\(', '['), '\\)', ']')") assertEquals("J[o]hn Do[e]", RecordPath.compile("replaceRegex(replaceRegex(/name, '\\(', '['), '\\)', ']')")
.evaluate(record).getSelectedFields().findFirst().get().getValue()); .evaluate(record).getSelectedFields().findFirst().get().getValue(),
"Brackets can be escaped with back-slash");
} }
@Test @Test
@ -1629,8 +1641,8 @@ public class TestRecordPath {
RecordPath.compile("base64Encode(/firstName)").evaluate(record).getSelectedFields().findFirst().get().getValue()); RecordPath.compile("base64Encode(/firstName)").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertEquals(Base64.getEncoder().encodeToString("Doe".getBytes(StandardCharsets.UTF_8)), assertEquals(Base64.getEncoder().encodeToString("Doe".getBytes(StandardCharsets.UTF_8)),
RecordPath.compile("base64Encode(/lastName)").evaluate(record).getSelectedFields().findFirst().get().getValue()); RecordPath.compile("base64Encode(/lastName)").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertTrue(Arrays.equals(Base64.getEncoder().encode("xyz".getBytes(StandardCharsets.UTF_8)), assertArrayEquals(Base64.getEncoder().encode("xyz".getBytes(StandardCharsets.UTF_8)),
(byte[]) RecordPath.compile("base64Encode(/b)").evaluate(record).getSelectedFields().findFirst().get().getValue())); (byte[]) RecordPath.compile("base64Encode(/b)").evaluate(record).getSelectedFields().findFirst().get().getValue());
List<Object> actualValues = RecordPath.compile("base64Encode(/*)").evaluate(record).getSelectedFields().map(FieldValue::getValue).collect(Collectors.toList()); List<Object> actualValues = RecordPath.compile("base64Encode(/*)").evaluate(record).getSelectedFields().map(FieldValue::getValue).collect(Collectors.toList());
IntStream.range(0, 3).forEach(i -> { IntStream.range(0, 3).forEach(i -> {
Object expectedObject = expectedValues.get(i); Object expectedObject = expectedValues.get(i);
@ -1638,7 +1650,7 @@ public class TestRecordPath {
if (actualObject instanceof String) { if (actualObject instanceof String) {
assertEquals(expectedObject, actualObject); assertEquals(expectedObject, actualObject);
} else if (actualObject instanceof byte[]) { } else if (actualObject instanceof byte[]) {
assertTrue(Arrays.equals((byte[]) expectedObject, (byte[]) actualObject)); assertArrayEquals((byte[]) expectedObject, (byte[]) actualObject);
} }
}); });
} }
@ -1660,7 +1672,7 @@ public class TestRecordPath {
assertEquals("John", RecordPath.compile("base64Decode(/firstName)").evaluate(record).getSelectedFields().findFirst().get().getValue()); assertEquals("John", RecordPath.compile("base64Decode(/firstName)").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertEquals("Doe", RecordPath.compile("base64Decode(/lastName)").evaluate(record).getSelectedFields().findFirst().get().getValue()); assertEquals("Doe", RecordPath.compile("base64Decode(/lastName)").evaluate(record).getSelectedFields().findFirst().get().getValue());
assertTrue(Arrays.equals("xyz".getBytes(StandardCharsets.UTF_8), (byte[]) RecordPath.compile("base64Decode(/b)").evaluate(record).getSelectedFields().findFirst().get().getValue())); assertArrayEquals("xyz".getBytes(StandardCharsets.UTF_8), (byte[]) RecordPath.compile("base64Decode(/b)").evaluate(record).getSelectedFields().findFirst().get().getValue());
List<Object> actualValues = RecordPath.compile("base64Decode(/*)").evaluate(record).getSelectedFields().map(FieldValue::getValue).collect(Collectors.toList()); List<Object> actualValues = RecordPath.compile("base64Decode(/*)").evaluate(record).getSelectedFields().map(FieldValue::getValue).collect(Collectors.toList());
IntStream.range(0, 3).forEach(i -> { IntStream.range(0, 3).forEach(i -> {
Object expectedObject = expectedValues.get(i); Object expectedObject = expectedValues.get(i);
@ -1668,7 +1680,7 @@ public class TestRecordPath {
if (actualObject instanceof String) { if (actualObject instanceof String) {
assertEquals(expectedObject, actualObject); assertEquals(expectedObject, actualObject);
} else if (actualObject instanceof byte[]) { } else if (actualObject instanceof byte[]) {
assertTrue(Arrays.equals((byte[]) expectedObject, (byte[]) actualObject)); assertArrayEquals((byte[]) expectedObject, (byte[]) actualObject);
} }
}); });
} }
@ -1733,8 +1745,8 @@ public class TestRecordPath {
new RecordField("json_str", RecordFieldType.STRING.getDataType()) new RecordField("json_str", RecordFieldType.STRING.getDataType())
)); ));
// test CHOICE resulting in nested ARRAY of RECORDs // test CHOICE resulting in nested ARRAY of Records
final Record recordAddressesArray = new MapRecord(schema, final Record mapAddressesArray = new MapRecord(schema,
Collections.singletonMap( Collections.singletonMap(
"json_str", "json_str",
"{\"firstName\":\"John\",\"age\":30,\"nicknames\":[\"J\",\"Johnny\"],\"addresses\":[{\"address_1\":\"123 Somewhere Street\"},{\"address_1\":\"456 Anywhere Road\"}]}") "{\"firstName\":\"John\",\"age\":30,\"nicknames\":[\"J\",\"Johnny\"],\"addresses\":[{\"address_1\":\"123 Somewhere Street\"},{\"address_1\":\"456 Anywhere Road\"}]}")
@ -1749,11 +1761,11 @@ public class TestRecordPath {
Collections.singletonMap("address_1", "456 Anywhere Road") Collections.singletonMap("address_1", "456 Anywhere Road")
)); ));
}}, }},
RecordPath.compile("unescapeJson(/json_str)").evaluate(recordAddressesArray).getSelectedFields().findFirst().orElseThrow(IllegalStateException::new).getValue() RecordPath.compile("unescapeJson(/json_str)").evaluate(mapAddressesArray).getSelectedFields().findFirst().orElseThrow(IllegalStateException::new).getValue()
); );
// test CHOICE resulting in nested single RECORD // test CHOICE resulting in nested single RECORD
final Record recordAddressesSingle = new MapRecord(schema, final Record mapAddressesSingle = new MapRecord(schema,
Collections.singletonMap( Collections.singletonMap(
"json_str", "json_str",
"{\"firstName\":\"John\",\"age\":30,\"nicknames\":[\"J\",\"Johnny\"],\"addresses\":{\"address_1\":\"123 Somewhere Street\"}}") "{\"firstName\":\"John\",\"age\":30,\"nicknames\":[\"J\",\"Johnny\"],\"addresses\":{\"address_1\":\"123 Somewhere Street\"}}")
@ -1765,7 +1777,35 @@ public class TestRecordPath {
put("nicknames", Arrays.asList("J", "Johnny")); put("nicknames", Arrays.asList("J", "Johnny"));
put("addresses", Collections.singletonMap("address_1", "123 Somewhere Street")); put("addresses", Collections.singletonMap("address_1", "123 Somewhere Street"));
}}, }},
RecordPath.compile("unescapeJson(/json_str)").evaluate(recordAddressesSingle).getSelectedFields().findFirst().orElseThrow(IllegalStateException::new).getValue() RecordPath.compile("unescapeJson(/json_str, 'false')").evaluate(mapAddressesSingle).getSelectedFields().findFirst().orElseThrow(IllegalStateException::new).getValue()
);
// test single Record converted from Map Object
final Record recordFromMap = new MapRecord(schema,
Collections.singletonMap(
"json_str",
"{\"firstName\":\"John\",\"age\":30}")
);
assertEquals(
DataTypeUtils.toRecord(new HashMap<String, Object>(){{
put("firstName", "John");
put("age", 30);
}}, "json_str"),
RecordPath.compile("unescapeJson(/json_str, 'true')").evaluate(recordFromMap).getSelectedFields().findFirst().orElseThrow(IllegalStateException::new).getValue()
);
// test collection of Record converted from Map collection
final Record recordCollectionFromMaps = new MapRecord(schema,
Collections.singletonMap(
"json_str",
"[{\"address_1\":\"123 Somewhere Street\"},{\"address_1\":\"456 Anywhere Road\"}]")
);
assertEquals(
Arrays.asList(
DataTypeUtils.toRecord(Collections.singletonMap("address_1", "123 Somewhere Street"), "json_str"),
DataTypeUtils.toRecord(Collections.singletonMap("address_1", "456 Anywhere Road"), "json_str")
),
RecordPath.compile("unescapeJson(/json_str, 'true')").evaluate(recordCollectionFromMaps).getSelectedFields().findFirst().orElseThrow(IllegalStateException::new).getValue()
); );
// test simple String field // test simple String field

View File

@ -893,7 +893,10 @@ The following record path expression would convert the record into an escaped JS
=== unescapeJson === unescapeJson
Converts a stringified JSON element to a Record, Array or simple field (e.g. String), using the UTF-8 character set. For example, given a schema such as: Converts a stringified JSON element to a Record, Array or simple field (e.g. String), using the UTF-8 character set.
Optionally convert JSON Objects parsed as Maps into Records (defaults to false).
For example, given a schema such as:
---- ----
{ {
@ -927,6 +930,23 @@ The following record path expression would populate the record with unescaped JS
Given a record such as: Given a record such as:
----
{
"json_str": "{\"name\":\"John\",\"age\":30}"
}
----
The following record path expression would return:
|==========================================================
| RecordPath | Return value
| `unescapeJson(/json_str, 'true')` | {"name": "John", "age": 30} (as a Record)
| `unescapeJson(/json_str, 'false')` | {"name"="John", "age"=30} (as a Map)
| `unescapeJson(/json_str)` | {"name"="John", "age"=30} (as a Map)
|==========================================================
Given a record such as:
---- ----
{ {
"json_str": "\"John\"" "json_str": "\"John\""

View File

@ -661,23 +661,29 @@
<exclude>src/test/resources/TestExtractGrok/apache.log</exclude> <exclude>src/test/resources/TestExtractGrok/apache.log</exclude>
<exclude>src/test/resources/TestExtractGrok/simple_text.log</exclude> <exclude>src/test/resources/TestExtractGrok/simple_text.log</exclude>
<exclude>src/test/resources/TestExtractGrok/patterns</exclude> <exclude>src/test/resources/TestExtractGrok/patterns</exclude>
<exclude>src/test/resources/TestUpdateRecord/input/embedded-string.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/input/person.json</exclude> <exclude>src/test/resources/TestUpdateRecord/input/person.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/input/person-address.json</exclude> <exclude>src/test/resources/TestUpdateRecord/input/person-address.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/input/person-stringified-name.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/input/person-with-null-array.json</exclude> <exclude>src/test/resources/TestUpdateRecord/input/person-with-null-array.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/input/multi-arrays.json</exclude> <exclude>src/test/resources/TestUpdateRecord/input/multi-arrays.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/embedded-string.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/person-with-null-array.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/person-with-null-array.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/person-with-firstname.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/person-with-firstname.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/person-with-capital-lastname.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/person-with-capital-lastname.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/name-fields-only.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/name-fields-only.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/name-and-mother-same.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/name-and-mother-same.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/person-with-name.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/person-with-new-city.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/person-with-new-city.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/embedded-record.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/person-with-address.avsc</exclude> <exclude>src/test/resources/TestUpdateRecord/schema/person-with-address.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc</exclude> <exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc</exclude> <exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc</exclude> <exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/name-fields-only.avsc</exclude> <exclude>src/test/resources/TestUpdateRecord/schema/name-fields-only.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-and-mother.avsc</exclude> <exclude>src/test/resources/TestUpdateRecord/schema/person-with-name-and-mother.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/person-with-stringified-name.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/schema/multi-arrays.avsc</exclude> <exclude>src/test/resources/TestUpdateRecord/schema/multi-arrays.avsc</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/updateArrays/multi-arrays-0and1.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/updateArrays/multi-arrays-0and1.json</exclude>
<exclude>src/test/resources/TestUpdateRecord/output/updateArrays/multi-arrays-0and2.json</exclude> <exclude>src/test/resources/TestUpdateRecord/output/updateArrays/multi-arrays-0and2.json</exclude>

View File

@ -201,32 +201,40 @@ public class UpdateRecord extends AbstractRecordProcessor {
} }
private Record processAbsolutePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, final Record record) { private Record processAbsolutePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, final Record record) {
final RecordPathResult replacementResult = replacementRecordPath.evaluate(record); final List<FieldValue> selectedFields = getSelectedFields(replacementRecordPath, null, record);
final List<FieldValue> selectedFields = replacementResult.getSelectedFields().collect(Collectors.toList());
final List<FieldValue> destinationFieldValues = destinationFields.collect(Collectors.toList()); final List<FieldValue> destinationFieldValues = destinationFields.collect(Collectors.toList());
return updateRecord(destinationFieldValues, selectedFields, record); return updateRecord(destinationFieldValues, selectedFields, record);
} }
private boolean isReplacingRoot(final List<FieldValue> destinationFields) {
return destinationFields.size() == 1 && !destinationFields.get(0).getParentRecord().isPresent();
}
private Record processRelativePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, Record record) { private Record processRelativePath(final RecordPath replacementRecordPath, final Stream<FieldValue> destinationFields, Record record) {
final List<FieldValue> destinationFieldValues = destinationFields.collect(Collectors.toList()); final List<FieldValue> destinationFieldValues = destinationFields.collect(Collectors.toList());
if (isReplacingRoot(destinationFieldValues)) {
final List<FieldValue> selectedFields = getSelectedFields(replacementRecordPath, destinationFieldValues.get(0), record);
record = updateRecord(destinationFieldValues, selectedFields, record);
} else {
for (final FieldValue fieldVal : destinationFieldValues) { for (final FieldValue fieldVal : destinationFieldValues) {
final RecordPathResult replacementResult = replacementRecordPath.evaluate(record, fieldVal); final List<FieldValue> selectedFields = getSelectedFields(replacementRecordPath, fieldVal, record);
final List<FieldValue> selectedFields = replacementResult.getSelectedFields().collect(Collectors.toList());
final Object replacementObject = getReplacementObject(selectedFields); final Object replacementObject = getReplacementObject(selectedFields);
updateFieldValue(fieldVal, replacementObject); updateFieldValue(fieldVal, replacementObject);
} }
}
return record; return record;
} }
private Record updateRecord(final List<FieldValue> destinationFields, final List<FieldValue> selectedFields, final Record record) { private Record updateRecord(final List<FieldValue> destinationFields, final List<FieldValue> selectedFields, final Record record) {
if (destinationFields.size() == 1 && !destinationFields.get(0).getParentRecord().isPresent()) { if (isReplacingRoot(destinationFields)) {
final Object replacement = getReplacementObject(selectedFields); final Object replacement = getReplacementObject(selectedFields);
if (replacement == null) { if (replacement == null) {
return record; return record;
} }
if (replacement instanceof Record) { if (replacement instanceof Record) {
return (Record) replacement; return (Record) replacement;
} }
@ -262,6 +270,11 @@ public class UpdateRecord extends AbstractRecordProcessor {
} }
} }
private List<FieldValue> getSelectedFields(final RecordPath replacementRecordPath, final FieldValue fieldValue, final Record record) {
final RecordPathResult replacementResult = replacementRecordPath.evaluate(record, fieldValue);
return replacementResult.getSelectedFields().collect(Collectors.toList());
}
private Object getReplacementObject(final List<FieldValue> selectedFields) { private Object getReplacementObject(final List<FieldValue> selectedFields) {
if (selectedFields.size() > 1) { if (selectedFields.size() > 1) {
final List<RecordField> fields = selectedFields.stream().map(FieldValue::getField).collect(Collectors.toList()); final List<RecordField> fields = selectedFields.stream().map(FieldValue::getField).collect(Collectors.toList());

View File

@ -438,6 +438,94 @@ public class TestUpdateRecord {
runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput); runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput);
} }
@Test
public void testSetRootWithUnescapeJsonCall() throws InitializationException, IOException {
final JsonTreeReader jsonReader = new JsonTreeReader();
runner.addControllerService("reader", jsonReader);
final String inputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-stringified-name.avsc")));
final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/name-fields-only.avsc")));
runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, inputSchemaText);
runner.enableControllerService(jsonReader);
final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
runner.addControllerService("writer", jsonWriter);
runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
runner.setProperty(jsonWriter, "Pretty Print JSON", "true");
runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute");
runner.enableControllerService(jsonWriter);
runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/person-stringified-name.json"));
runner.setProperty("/", "unescapeJson(/stringified_name, 'true')");
runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES);
runner.run();
runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
final String expectedOutput = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/output/name-fields-only.json")));
runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput);
}
@Test
public void testSetFieldWithUnescapeJsonCall() throws InitializationException, IOException {
final JsonTreeReader jsonReader = new JsonTreeReader();
runner.addControllerService("reader", jsonReader);
final String inputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-stringified-name.avsc")));
final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc")));
runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, inputSchemaText);
runner.enableControllerService(jsonReader);
final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
runner.addControllerService("writer", jsonWriter);
runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
runner.setProperty(jsonWriter, "Pretty Print JSON", "true");
runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute");
runner.enableControllerService(jsonWriter);
runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/person-stringified-name.json"));
runner.setProperty("/name", "unescapeJson(/stringified_name)");
runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES);
runner.run();
runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
final String expectedOutput = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/output/person-with-name.json")));
runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput);
}
@Test
public void testSetNestedRecordWithUnescapeJsonCall() throws InitializationException, IOException {
final JsonTreeReader jsonReader = new JsonTreeReader();
runner.addControllerService("reader", jsonReader);
final String schemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/embedded-record.avsc")));
runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, schemaText);
runner.enableControllerService(jsonReader);
final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
runner.addControllerService("writer", jsonWriter);
runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, schemaText);
runner.setProperty(jsonWriter, "Pretty Print JSON", "true");
runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute");
runner.enableControllerService(jsonWriter);
runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/embedded-string.json"));
runner.setProperty("/embedded", "unescapeJson(/str)");
runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.RECORD_PATH_VALUES);
runner.run();
runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1);
final String expectedOutput = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/output/embedded-record.json")));
runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput);
}
@Test @Test
public void testSetRootPathRelativeWithMultipleValues() throws InitializationException, IOException { public void testSetRootPathRelativeWithMultipleValues() throws InitializationException, IOException {

View File

@ -0,0 +1,4 @@
{
"str": "{\"label\":\"Test!\",\"child\":{\"name\":\"Child record!\"}}",
"embedded": null
}

View File

@ -0,0 +1,4 @@
{
"id": 485,
"stringified_name": "{\"last\": \"Doe\", \"first\": \"John\"}"
}

View File

@ -0,0 +1,9 @@
[ {
"str" : "{\"label\":\"Test!\",\"child\":{\"name\":\"Child record!\"}}",
"embedded" : {
"label" : "Test!",
"child" : {
"name" : "Child record!"
}
}
} ]

View File

@ -0,0 +1,7 @@
[ {
"id" : 485,
"name" : {
"last" : "Doe",
"first" : "John"
}
} ]

View File

@ -0,0 +1,39 @@
{
"name": "Parent",
"type": "record",
"fields": [
{
"name": "str",
"type": "string"
},
{
"name": "embedded",
"type": [
{
"name": "EmbeddedRecord",
"type": "record",
"fields": [
{
"name": "label",
"type": "string"
},
{
"name": "child",
"type": {
"name": "ChildRecord",
"type": "record",
"fields": [
{
"name": "name",
"type": "string"
}
]
}
}
]
},
"null"
]
}
]
}

View File

@ -0,0 +1,9 @@
{
"name": "personWithNameRecord",
"namespace": "nifi",
"type": "record",
"fields": [
{ "name": "id", "type": "int" },
{ "name": "stringified_name", "type": "string" }
]
}