Jr 20241008 merge data absent reason (#6370)

* handle single cardinality case

* multiple cardinality case

* change log

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/TerserUtil.java

@@ -26,10 +26,12 @@ import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeChildChoiceDefinition;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
 import ca.uhn.fhir.i18n.Msg;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.tuple.Triple;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseBackboneElement;
+import org.hl7.fhir.instance.model.api.IBaseHasExtensions;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 import org.slf4j.Logger;
@@ -93,6 +95,8 @@ public final class TerserUtil {
 
 	private static final Logger ourLog = getLogger(TerserUtil.class);
 	private static final String EQUALS_DEEP = "equalsDeep";
+	public static final String DATA_ABSENT_REASON_EXTENSION_URI =
+			"http://hl7.org/fhir/StructureDefinition/data-absent-reason";
 
 	private TerserUtil() {}
 
@@ -266,6 +270,15 @@ public final class TerserUtil {
 		return theItems.stream().anyMatch(i -> equals(i, theItem, method));
 	}
 
+	private static boolean hasDataAbsentReason(IBase theItem) {
+		if (theItem instanceof IBaseHasExtensions) {
+			IBaseHasExtensions hasExtensions = (IBaseHasExtensions) theItem;
+			return hasExtensions.getExtension().stream()
+					.anyMatch(t -> StringUtils.equals(t.getUrl(), DATA_ABSENT_REASON_EXTENSION_URI));
+		}
+		return false;
+	}
+
 	/**
 	 * Merges all fields on the provided instance. <code>theTo</code> will contain a union of all values from <code>theFrom</code>
 	 * instance and <code>theTo</code> instance.
@@ -695,24 +708,36 @@ public final class TerserUtil {
 			BaseRuntimeChildDefinition childDefinition,
 			List<IBase> theFromFieldValues,
 			List<IBase> theToFieldValues) {
-		for (IBase theFromFieldValue : theFromFieldValues) {
+		if (!theFromFieldValues.isEmpty() && theToFieldValues.stream().anyMatch(TerserUtil::hasDataAbsentReason)) {
-			if (contains(theFromFieldValue, theToFieldValues)) {
+			// If the to resource has a data absent reason, and there is potentially real data incoming
+			// in the from resource, we should clear the data absent reason because it won't be absent anymore.
+			theToFieldValues = removeDataAbsentReason(theTo, childDefinition, theToFieldValues);
+		}
+
+		for (IBase fromFieldValue : theFromFieldValues) {
+			if (contains(fromFieldValue, theToFieldValues)) {
 				continue;
 			}
 
-			IBase newFieldValue = newElement(theTerser, childDefinition, theFromFieldValue, null);
+			if (hasDataAbsentReason(fromFieldValue) && !theToFieldValues.isEmpty()) {
-			if (theFromFieldValue instanceof IPrimitiveType) {
+				// if the from field value asserts a reason the field isn't populated, but the to field is populated,
+				// we don't want to overwrite real data with the extension
+				continue;
+			}
+
+			IBase newFieldValue = newElement(theTerser, childDefinition, fromFieldValue, null);
+			if (fromFieldValue instanceof IPrimitiveType) {
 				try {
-					Method copyMethod = getMethod(theFromFieldValue, "copy");
+					Method copyMethod = getMethod(fromFieldValue, "copy");
 					if (copyMethod != null) {
-						newFieldValue = (IBase) copyMethod.invoke(theFromFieldValue, new Object[] {});
+						newFieldValue = (IBase) copyMethod.invoke(fromFieldValue, new Object[] {});
 					}
 				} catch (Throwable t) {
-					((IPrimitiveType) newFieldValue)
+					((IPrimitiveType<?>) newFieldValue)
-							.setValueAsString(((IPrimitiveType) theFromFieldValue).getValueAsString());
+							.setValueAsString(((IPrimitiveType<?>) fromFieldValue).getValueAsString());
 				}
 			} else {
-				theTerser.cloneInto(theFromFieldValue, newFieldValue, true);
+				theTerser.cloneInto(fromFieldValue, newFieldValue, true);
 			}
 
 			try {
@@ -724,6 +749,21 @@ public final class TerserUtil {
 		}
 	}
 
+	private static List<IBase> removeDataAbsentReason(
+			IBaseResource theResource, BaseRuntimeChildDefinition theFieldDefinition, List<IBase> theFieldValues) {
+		for (int i = 0; i < theFieldValues.size(); i++) {
+			if (hasDataAbsentReason(theFieldValues.get(i))) {
+				try {
+					theFieldDefinition.getMutator().remove(theResource, i);
+				} catch (UnsupportedOperationException e) {
+					// the field must be single-valued, just clear it
+					theFieldDefinition.getMutator().setValue(theResource, null);
+				}
+			}
+		}
+		return theFieldDefinition.getAccessor().getValues(theResource);
+	}
+
 	/**
 	 * Clones the specified resource.
 	 *

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_6_0/6370-data-absent-merge.yaml

@@ -0,0 +1,6 @@
+---
+type: add
+issue: 6370
+title: "When using the FHIR `TerserUtil` to merge two resource, if one resource has real data in a particular field,
+and the other resource has a `data-absent-reason` extension in the same field, the real data will be given
+precedence in the merged resource, and the extension will be ignored."

hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/util/TerserUtilTest.java

@@ -2,6 +2,7 @@ package ca.uhn.fhir.util;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.RuntimeResourceDefinition;
+import org.apache.commons.lang3.StringUtils;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseBackboneElement;
 import org.hl7.fhir.r4.model.Address;
@@ -10,10 +11,12 @@ import org.hl7.fhir.r4.model.Claim;
 import org.hl7.fhir.r4.model.Condition;
 import org.hl7.fhir.r4.model.DateTimeType;
 import org.hl7.fhir.r4.model.DateType;
+import org.hl7.fhir.r4.model.Enumeration;
 import org.hl7.fhir.r4.model.Enumerations;
 import org.hl7.fhir.r4.model.Extension;
 import org.hl7.fhir.r4.model.HumanName;
 import org.hl7.fhir.r4.model.Identifier;
+import org.hl7.fhir.r4.model.Observation;
 import org.hl7.fhir.r4.model.Organization;
 import org.hl7.fhir.r4.model.Patient;
 import org.hl7.fhir.r4.model.Practitioner;
@@ -21,9 +24,14 @@ import org.hl7.fhir.r4.model.PrimitiveType;
 import org.hl7.fhir.r4.model.Reference;
 import org.hl7.fhir.r4.model.StringType;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.util.Date;
+import java.util.List;
 import java.util.UUID;
+import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -118,6 +126,7 @@ class TerserUtilTest {
 			            ]
 			      }
 			  	""";
+	public static final String DATA_ABSENT_REASON_EXTENSION_URI = "http://hl7.org/fhir/StructureDefinition/data-absent-reason";
 
 	@Test
 	void cloneIdentifierIntoResource() {
@@ -417,6 +426,102 @@ class TerserUtilTest {
 		assertThat(c2.getRecorder().getResource()).isSameAs(practitioner);
 	}
 
+	@ParameterizedTest
+	@MethodSource("singleCardinalityArguments")
+	public void testMergeWithDataAbsentReason_singleCardinality(
+		 Enumeration<Observation.ObservationStatus> theFromStatus,
+		 Enumeration<Observation.ObservationStatus> theToStatus,
+		 Enumeration<Observation.ObservationStatus> theExpectedStatus) {
+		Observation fromObservation = new Observation();
+		fromObservation.setStatusElement(theFromStatus);
+
+		Observation toObservation = new Observation();
+		toObservation.setStatusElement(theToStatus);
+
+		TerserUtil.mergeField(ourFhirContext, "status", fromObservation, toObservation);
+
+		if (theExpectedStatus == null) {
+			assertThat(toObservation.hasStatus()).isFalse();
+		} else {
+			assertThat(toObservation.getStatusElement().getCode()).isEqualTo(theExpectedStatus.getCode());
+		}
+	}
+
+	private static Stream<Arguments> singleCardinalityArguments() {
+		return Stream.of(
+			 Arguments.of(null, null, null),
+			 Arguments.of(statusFromEnum(Observation.ObservationStatus.FINAL), null, statusFromEnum(Observation.ObservationStatus.FINAL)),
+			 Arguments.of(null, statusFromEnum(Observation.ObservationStatus.FINAL), statusFromEnum(Observation.ObservationStatus.FINAL)),
+			 Arguments.of(statusFromEnum(Observation.ObservationStatus.FINAL), statusFromEnum(Observation.ObservationStatus.PRELIMINARY), statusFromEnum(Observation.ObservationStatus.FINAL)),
+			 Arguments.of(statusWithDataAbsentReason(), null, statusWithDataAbsentReason()),
+			 Arguments.of(null, statusWithDataAbsentReason(), statusWithDataAbsentReason()),
+			 Arguments.of(statusWithDataAbsentReason(), statusWithDataAbsentReason(), statusWithDataAbsentReason()),
+			 Arguments.of(statusFromEnum(Observation.ObservationStatus.FINAL), statusWithDataAbsentReason(), statusFromEnum(Observation.ObservationStatus.FINAL)),
+			 Arguments.of(statusWithDataAbsentReason(), statusFromEnum(Observation.ObservationStatus.FINAL), statusFromEnum(Observation.ObservationStatus.FINAL))
+		);
+	}
+
+	private static Enumeration<Observation.ObservationStatus> statusFromEnum(Observation.ObservationStatus theStatus) {
+		return new Enumeration<>(new Observation.ObservationStatusEnumFactory(), theStatus);
+	}
+
+	private static Enumeration<Observation.ObservationStatus> statusWithDataAbsentReason() {
+		Enumeration<Observation.ObservationStatus> enumeration = new Enumeration<>(new Observation.ObservationStatusEnumFactory());
+		Enumeration<Enumerations.DataAbsentReason> extension = new Enumeration<>(new Enumerations.DataAbsentReasonEnumFactory(), Enumerations.DataAbsentReason.UNKNOWN);
+		enumeration.addExtension(DATA_ABSENT_REASON_EXTENSION_URI, extension);
+		return enumeration;
+	}
+
+	@ParameterizedTest
+	@MethodSource("multipleCardinalityArguments")
+	public void testMergeWithDataAbsentReason_multipleCardinality(
+		 List<Identifier> theFromIdentifiers, List<Identifier> theToIdentifiers, List<Identifier> theExpectedIdentifiers) {
+		Observation fromObservation = new Observation();
+		theFromIdentifiers.forEach(fromObservation::addIdentifier);
+
+		Observation toObservation = new Observation();
+		theToIdentifiers.forEach(toObservation::addIdentifier);
+
+		TerserUtil.mergeField(ourFhirContext, "identifier", fromObservation, toObservation);
+
+		assertThat(toObservation.getIdentifier()).hasSize(theExpectedIdentifiers.size());
+		assertThat(toObservation.getIdentifier()).allMatch(t -> {
+			if (t.hasValue()) {
+				return theExpectedIdentifiers.stream().anyMatch(s -> StringUtils.equals(t.getValue(), s.getValue()));
+			} else if (t.hasExtension(DATA_ABSENT_REASON_EXTENSION_URI)) {
+				return theExpectedIdentifiers.stream().anyMatch(s -> s.hasExtension(DATA_ABSENT_REASON_EXTENSION_URI));
+			}
+			return false;
+		});
+	}
+
+	private static Stream<Arguments> multipleCardinalityArguments() {
+		return Stream.of(
+			 Arguments.of(List.of(), List.of(), List.of()),
+			 Arguments.of(List.of(identifierFromValue("identifier1")), List.of(), List.of(identifierFromValue("identifier1"))),
+			 Arguments.of(List.of(), List.of(identifierFromValue("identifier1")), List.of(identifierFromValue("identifier1"))),
+			 Arguments.of(List.of(identifierFromValue("identifier1")), List.of(identifierFromValue("identifier2")), List.of(identifierFromValue("identifier1"), identifierFromValue("identifier2"))),
+			 Arguments.of(List.of(identifierWithDataAbsentReason()), List.of(), List.of(identifierWithDataAbsentReason())),
+			 Arguments.of(List.of(), List.of(identifierWithDataAbsentReason()), List.of(identifierWithDataAbsentReason())),
+			 Arguments.of(List.of(identifierWithDataAbsentReason()), List.of(identifierWithDataAbsentReason()), List.of(identifierWithDataAbsentReason())),
+			 Arguments.of(List.of(identifierFromValue("identifier1")), List.of(identifierWithDataAbsentReason()), List.of(identifierFromValue("identifier1"))),
+			 Arguments.of(List.of(identifierWithDataAbsentReason()), List.of(identifierFromValue("identifier1")), List.of(identifierFromValue("identifier1"))),
+			 Arguments.of(List.of(identifierFromValue("identifier1"), identifierFromValue("identifier2")), List.of(identifierWithDataAbsentReason()), List.of(identifierFromValue("identifier1"), identifierFromValue("identifier2"))),
+			 Arguments.of(List.of(identifierWithDataAbsentReason()), List.of(identifierFromValue("identifier1"), identifierFromValue("identifier2")), List.of(identifierFromValue("identifier1"), identifierFromValue("identifier2")))
+		);
+	}
+
+	private static Identifier identifierFromValue(String theValue) {
+		return new Identifier().setValue(theValue);
+	}
+
+	private static Identifier identifierWithDataAbsentReason() {
+		Identifier identifier = new Identifier();
+		Enumeration<Enumerations.DataAbsentReason> extension = new Enumeration<>(new Enumerations.DataAbsentReasonEnumFactory(), Enumerations.DataAbsentReason.UNKNOWN);
+		identifier.addExtension(DATA_ABSENT_REASON_EXTENSION_URI, extension);
+		return identifier;
+	}
+
 	@Test
 	void testCloneWithDuplicateNonPrimitives() {
 		Patient p1 = new Patient();