From 1191e511a5306d6c23ed0b289db85c11c0373690 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Tue, 26 Oct 2021 10:05:31 -0400 Subject: [PATCH] NIFI-9335: Updated AvroTypeUtil#createAvroRecord to ensure that if the given Avro Schema contains a field whose value is defaulted, the produced Avro Record has that value populated. Also added a unit test to verify behavior. Ran performance test (which is igonred and must be manually enabled) - about 2.1 seconds before the change and about 2.5 seconds after the change for each iteration. Signed-off-by: Matthew Burgess This closes #5483 --- .../org/apache/nifi/avro/AvroTypeUtil.java | 13 +++++++++++ .../apache/nifi/avro/TestAvroTypeUtil.java | 22 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java index 286b418be7..01f4fa7552 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java @@ -619,6 +619,19 @@ public class AvroTypeUtil { rec.put(fieldName, converted); } + // see if the Avro schema has any fields that aren't in the RecordSchema, and if those fields have a default + // value then we want to populate it in the GenericRecord being produced + for (final Field field : avroSchema.getFields()) { + if (field.defaultVal() == null) { + continue; + } + + final Optional recordField = recordSchema.getField(field.name()); + if (!recordField.isPresent() && rec.get(field.name()) == null) { + rec.put(field.name(), field.defaultVal()); + } + } + return rec; } diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java index bc2e65f4f3..05d31a0237 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java @@ -23,6 +23,7 @@ import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; import org.apache.avro.Schema.Type; +import org.apache.avro.SchemaBuilder; import org.apache.avro.file.DataFileStream; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; @@ -31,7 +32,6 @@ import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; import org.apache.avro.util.Utf8; -import org.apache.nifi.schema.access.SchemaNotFoundException; import org.apache.nifi.serialization.SimpleRecordSchema; import org.apache.nifi.serialization.record.DataType; import org.apache.nifi.serialization.record.MapRecord; @@ -105,7 +105,25 @@ public class TestAvroTypeUtil { } @Test - public void testCreateAvroSchemaPrimitiveTypes() throws SchemaNotFoundException { + public void testAvroDefaultValueWithNoFieldInRecordOrSchema() throws IOException { + final List fields = new ArrayList<>(); + fields.add(new RecordField("name", RecordFieldType.STRING.getDataType())); + final RecordSchema personSchema = new SimpleRecordSchema(fields); + + final org.apache.nifi.serialization.record.Record record = new MapRecord(personSchema, Collections.singletonMap("name", "John Doe")); + final Schema avroSchema = SchemaBuilder.record("person").namespace("nifi") + .fields() + .requiredString("name") + .name("color").type().stringType().stringDefault("blue") + .endRecord(); + + final GenericRecord avroRecord = AvroTypeUtil.createAvroRecord(record, avroSchema); + assertEquals("John Doe", avroRecord.get("name")); + assertEquals("blue", avroRecord.get("color")); + } + + @Test + public void testCreateAvroSchemaPrimitiveTypes() { final List fields = new ArrayList<>(); fields.add(new RecordField("int", RecordFieldType.INT.getDataType())); fields.add(new RecordField("long", RecordFieldType.LONG.getDataType()));