diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java index a0eea8b1c4..caa2743b06 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java @@ -626,7 +626,7 @@ public class AvroTypeUtil { recordFields.add(new RecordField(fieldName, dataType, field.aliases(), nullable)); } else { Object defaultValue = field.defaultVal(); - if (fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) { + if (defaultValue != null && fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) { defaultValue = defaultValue instanceof List ? ((List) defaultValue).toArray() : new Object[0]; } recordFields.add(new RecordField(fieldName, dataType, defaultValue, field.aliases(), nullable)); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 3da027ad03..72cdb6581f 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -570,6 +570,9 @@ src/test/resources/TestForkRecord/schema/schema.avsc src/test/resources/TestConvertRecord/schema/person.avsc src/test/resources/TestConvertRecord/input/person.json + src/test/resources/TestValidateRecord/missing-array.json + src/test/resources/TestValidateRecord/missing-array.avsc + src/test/resources/TestValidateRecord/missing-array-with-default.avsc src/test/resources/TestValidateRecord/nested-map-input.json src/test/resources/TestValidateRecord/nested-map-schema.avsc src/test/resources/TestValidateRecord/timestamp.avsc diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java index bbcc9a9369..f53dd5cd76 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java @@ -412,6 +412,80 @@ public class TestValidateRecord { runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0); } + @Test + public void testValidateMissingRequiredArray() throws InitializationException, IOException { + final String validateSchema = new String(Files.readAllBytes(Paths.get("src/test/resources/TestValidateRecord/missing-array.avsc")), StandardCharsets.UTF_8); + + final JsonTreeReader jsonReader = new JsonTreeReader(); + runner.addControllerService("reader", jsonReader); + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, "schema-text-property"); + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, validateSchema); + runner.enableControllerService(jsonReader); + + final JsonRecordSetWriter validWriter = new JsonRecordSetWriter(); + runner.addControllerService("writer", validWriter); + runner.setProperty(validWriter, "Schema Write Strategy", "full-schema-attribute"); + runner.enableControllerService(validWriter); + + final MockRecordWriter invalidWriter = new MockRecordWriter("invalid", true); + runner.addControllerService("invalid-writer", invalidWriter); + runner.enableControllerService(invalidWriter); + + runner.setProperty(ValidateRecord.RECORD_READER, "reader"); + runner.setProperty(ValidateRecord.RECORD_WRITER, "writer"); + runner.setProperty(ValidateRecord.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY); + runner.setProperty(ValidateRecord.SCHEMA_TEXT, validateSchema); + runner.setProperty(ValidateRecord.INVALID_RECORD_WRITER, "invalid-writer"); + runner.setProperty(ValidateRecord.ALLOW_EXTRA_FIELDS, "true"); + + // The record is invalid due to not containing the required array from the schema + runner.setProperty(ValidateRecord.STRICT_TYPE_CHECKING, "false"); + runner.enqueue(Paths.get("src/test/resources/TestValidateRecord/missing-array.json")); + runner.run(); + + runner.assertTransferCount(ValidateRecord.REL_VALID, 0); + runner.assertTransferCount(ValidateRecord.REL_INVALID, 1); + runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0); + runner.clearTransferState(); + } + + @Test + public void testValidateMissingRequiredArrayWithDefault() throws InitializationException, IOException { + final String validateSchema = new String(Files.readAllBytes(Paths.get("src/test/resources/TestValidateRecord/missing-array-with-default.avsc")), StandardCharsets.UTF_8); + + final JsonTreeReader jsonReader = new JsonTreeReader(); + runner.addControllerService("reader", jsonReader); + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, "schema-text-property"); + runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT, validateSchema); + runner.enableControllerService(jsonReader); + + final JsonRecordSetWriter validWriter = new JsonRecordSetWriter(); + runner.addControllerService("writer", validWriter); + runner.setProperty(validWriter, "Schema Write Strategy", "full-schema-attribute"); + runner.enableControllerService(validWriter); + + final MockRecordWriter invalidWriter = new MockRecordWriter("invalid", true); + runner.addControllerService("invalid-writer", invalidWriter); + runner.enableControllerService(invalidWriter); + + runner.setProperty(ValidateRecord.RECORD_READER, "reader"); + runner.setProperty(ValidateRecord.RECORD_WRITER, "writer"); + runner.setProperty(ValidateRecord.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY); + runner.setProperty(ValidateRecord.SCHEMA_TEXT, validateSchema); + runner.setProperty(ValidateRecord.INVALID_RECORD_WRITER, "invalid-writer"); + runner.setProperty(ValidateRecord.ALLOW_EXTRA_FIELDS, "true"); + + // The record is invalid due to not containing the required array from the schema + runner.setProperty(ValidateRecord.STRICT_TYPE_CHECKING, "false"); + runner.enqueue(Paths.get("src/test/resources/TestValidateRecord/missing-array.json")); + runner.run(); + + runner.assertTransferCount(ValidateRecord.REL_VALID, 1); + runner.assertTransferCount(ValidateRecord.REL_INVALID, 0); + runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0); + runner.clearTransferState(); + } + @Test public void testValidateJsonTimestamp() throws IOException, InitializationException { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array-with-default.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array-with-default.avsc new file mode 100644 index 0000000000..b01911c552 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array-with-default.avsc @@ -0,0 +1,22 @@ +{ + "name": "Test", + "type": "record", + "fields": [ + { + "name": "Records", + "type": { + "type": "array", + "items": { + "name": "Records_record", + "type": "record", + "fields": [ + { + "name": "id", + "type": "string" + } + ] + } + }, "default": [] + } + ] +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.avsc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.avsc new file mode 100644 index 0000000000..9175a4cb4b --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.avsc @@ -0,0 +1,22 @@ +{ + "name": "Test", + "type": "record", + "fields": [ + { + "name": "Records", + "type": { + "type": "array", + "items": { + "name": "Records_record", + "type": "record", + "fields": [ + { + "name": "id", + "type": "string" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.json b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.json new file mode 100644 index 0000000000..43b8c4c417 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/missing-array.json @@ -0,0 +1,5 @@ +{ + "NotRecords": [{ + "id": "1" + }] +}