From bdab3cda0a0552f663d3c0350240afc21c4906cd Mon Sep 17 00:00:00 2001 From: Pierre Villard Date: Fri, 1 Sep 2017 15:16:11 +0200 Subject: [PATCH] NIFI-4340 - fix record path evaluation when array is [ null ] This closes #2122. --- .../nifi/record/path/StandardFieldValue.java | 2 + .../record/path/paths/ChildFieldPath.java | 4 ++ .../nifi-standard-processors/pom.xml | 5 ++ .../processors/standard/TestUpdateRecord.java | 60 +++++++++++++++++++ .../input/person-address.json | 13 ++++ .../input/person-with-null-array.json | 8 +++ .../output/person-with-new-city.json | 11 ++++ .../output/person-with-null-array.json | 8 +++ .../schema/person-with-address.avsc | 29 +++++++++ 9 files changed, 140 insertions(+) create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-address.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-with-null-array.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-new-city.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-null-array.json create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-address.avsc diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java index 75644c504e..589708622c 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/StandardFieldValue.java @@ -122,6 +122,8 @@ public class StandardFieldValue implements FieldValue { if (value instanceof Record) { ((Record) value).setValue(getField().getFieldName(), newValue); return; + } else if (value == null) { + return; // value is null, nothing to update } else { throw new UnsupportedOperationException("Cannot update the field value because the value is not associated with any record"); } diff --git a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java index 93f3d6558e..ac0385a5fc 100644 --- a/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java +++ b/nifi-commons/nifi-record-path/src/main/java/org/apache/nifi/record/path/paths/ChildFieldPath.java @@ -47,6 +47,10 @@ public class ChildFieldPath extends RecordPathSegment { } final Record record = (Record) fieldValue.getValue(); + if(record == null) { + return missingChild(fieldValue); + } + final Object value = record.getValue(childName); if (value == null) { return missingChild(fieldValue); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index bcaa99efd1..b2a9d4345d 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -498,11 +498,16 @@ src/test/resources/TestExtractGrok/simple_text.log src/test/resources/TestExtractGrok/patterns src/test/resources/TestUpdateRecord/input/person.json + src/test/resources/TestUpdateRecord/input/person-address.json + src/test/resources/TestUpdateRecord/input/person-with-null-array.json + src/test/resources/TestUpdateRecord/output/person-with-null-array.json src/test/resources/TestUpdateRecord/output/person-with-firstname.json src/test/resources/TestUpdateRecord/output/person-with-firstname-lastname.json src/test/resources/TestUpdateRecord/output/person-with-capital-lastname.json src/test/resources/TestUpdateRecord/output/name-fields-only.json src/test/resources/TestUpdateRecord/output/name-and-mother-same.json + src/test/resources/TestUpdateRecord/output/person-with-new-city.json + src/test/resources/TestUpdateRecord/schema/person-with-address.avsc src/test/resources/TestUpdateRecord/schema/person-with-name-record.avsc src/test/resources/TestUpdateRecord/schema/person-with-name-string.avsc src/test/resources/TestUpdateRecord/schema/person-with-name-string-fields.avsc diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java index aa675b8652..6669f4bdc2 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUpdateRecord.java @@ -184,6 +184,66 @@ public class TestUpdateRecord { runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput); } + @Test + public void testUpdateInArray() throws InitializationException, IOException { + final JsonTreeReader jsonReader = new JsonTreeReader(); + runner.addControllerService("reader", jsonReader); + + final String inputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-address.avsc"))); + final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-address.avsc"))); + + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY); + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, inputSchemaText); + runner.enableControllerService(jsonReader); + + final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter(); + runner.addControllerService("writer", jsonWriter); + runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY); + runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText); + runner.setProperty(jsonWriter, "Pretty Print JSON", "true"); + runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute"); + runner.enableControllerService(jsonWriter); + + runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/person-address.json")); + runner.setProperty("/address[*]/city", "newCity"); + runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.LITERAL_VALUES); + + runner.run(); + runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1); + final String expectedOutput = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/output/person-with-new-city.json"))); + runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput); + } + + @Test + public void testUpdateInNullArray() throws InitializationException, IOException { + final JsonTreeReader jsonReader = new JsonTreeReader(); + runner.addControllerService("reader", jsonReader); + + final String inputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-address.avsc"))); + final String outputSchemaText = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/schema/person-with-address.avsc"))); + + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY); + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, inputSchemaText); + runner.enableControllerService(jsonReader); + + final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter(); + runner.addControllerService("writer", jsonWriter); + runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY); + runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText); + runner.setProperty(jsonWriter, "Pretty Print JSON", "true"); + runner.setProperty(jsonWriter, "Schema Write Strategy", "full-schema-attribute"); + runner.enableControllerService(jsonWriter); + + runner.enqueue(Paths.get("src/test/resources/TestUpdateRecord/input/person-with-null-array.json")); + runner.setProperty("/address[*]/city", "newCity"); + runner.setProperty(UpdateRecord.REPLACEMENT_VALUE_STRATEGY, UpdateRecord.LITERAL_VALUES); + + runner.run(); + runner.assertAllFlowFilesTransferred(UpdateRecord.REL_SUCCESS, 1); + final String expectedOutput = new String(Files.readAllBytes(Paths.get("src/test/resources/TestUpdateRecord/output/person-with-null-array.json"))); + runner.getFlowFilesForRelationship(UpdateRecord.REL_SUCCESS).get(0).assertContentEquals(expectedOutput); + } + @Test public void testAddFieldNotInInputRecord() throws InitializationException, IOException { final JsonTreeReader jsonReader = new JsonTreeReader(); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-address.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-address.json new file mode 100644 index 0000000000..df00fd2af9 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-address.json @@ -0,0 +1,13 @@ +{ + "id": 485, + "name": { + "last": "Doe", + "first": "John" + }, + "address": [ + { + "street": "1 nifi stree", + "city": "nifi" + } + ] +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-with-null-array.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-with-null-array.json new file mode 100644 index 0000000000..61e20a9844 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/input/person-with-null-array.json @@ -0,0 +1,8 @@ +[ { + "id" : 485, + "name" : { + "last" : "Doe", + "first" : "John" + }, + "address" : [ null ] +} ] \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-new-city.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-new-city.json new file mode 100644 index 0000000000..552969b42e --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-new-city.json @@ -0,0 +1,11 @@ +[ { + "id" : 485, + "name" : { + "last" : "Doe", + "first" : "John" + }, + "address" : [ { + "street" : "1 nifi stree", + "city" : "newCity" + } ] +} ] \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-null-array.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-null-array.json new file mode 100644 index 0000000000..61e20a9844 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/output/person-with-null-array.json @@ -0,0 +1,8 @@ +[ { + "id" : 485, + "name" : { + "last" : "Doe", + "first" : "John" + }, + "address" : [ null ] +} ] \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-address.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-address.avsc new file mode 100644 index 0000000000..a076998231 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestUpdateRecord/schema/person-with-address.avsc @@ -0,0 +1,29 @@ +{ + "name": "personWithNameRecord", + "namespace": "nifi", + "type": "record", + "fields": [ + { "name": "id", "type": "int" }, + { "name": "name", "type": { + "type": "record", + "name": "nameRecord", + "fields": [ + { "name": "last", "type": "string" }, + { "name": "first", "type": "string" } + ] + } + }, + { "name" : "address", "type": ["null", + { "type" : "array", "items" : { + "type" : "record", + "name" : "Person", + "fields" : [ + { "name" : "street", "type": "string" }, + { "name" : "city", "type": "string" } + ] + } + } + ] + } + ] +} \ No newline at end of file