Survivorship 4

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmSurvivorshipSvcImpl.java

@@ -23,7 +23,9 @@ package ca.uhn.fhir.jpa.mdm.svc;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.mdm.api.IMdmSurvivorshipService;
 import ca.uhn.fhir.mdm.model.MdmTransactionContext;
+import ca.uhn.fhir.mdm.util.TerserUtil;
 import org.hl7.fhir.instance.model.api.IBase;
+import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.springframework.beans.factory.annotation.Autowired;
 
 public class MdmSurvivorshipSvcImpl implements IMdmSurvivorshipService {
@@ -31,8 +33,42 @@ public class MdmSurvivorshipSvcImpl implements IMdmSurvivorshipService {
 	@Autowired
 	private FhirContext myFhirContext;
 
+	/**
+	 * Survivorship rules may include the following data consolidation methods:
+	 *
+	 * <ul>
+	 *  <li>
+	 *  Length of field - apply the field value containing most or least number of characters - e.g. longest name
+	 *  </li>
+	 *  <li>
+	 *  Date time - all the field value from the oldest or the newest recrod - e.g. use the most recent phone number
+	 *  </li>
+	 *  <li>
+	 *  Frequency - use the most or least frequent number of occurrence - e.g. most common phone number
+	 *  </li>
+	 *  <li>
+	 *  Integer - number functions (largest, sum, avg) - e.g. number of patient encounters
+	 *  </li>
+	 *  <li>
+	 *  Quality of data - best quality data - e.g. data coming from a certain system is considered trusted and overrides all other values
+	 *  </li>
+	 *  <li>
+	 *  A hybrid approach combining all methods listed above as best fits
+	 *  </li>
+	 * </ul>
+	 *
+	 * @param theTargetResource        Target resource to merge fields from
+	 * @param theGoldenResource        Golden resource to merge fields into
+	 * @param theMdmTransactionContext Current transaction context
+	 * @param <T>
+	 */
 	@Override
 	public <T extends IBase> void applySurvivorshipRulesToGoldenResource(T theTargetResource, T theGoldenResource, MdmTransactionContext theMdmTransactionContext) {
-//		TerserUtil.cloneFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource);
+		// TerserUtil.mergeFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.DEFAULT_EXCLUDED_FIELDS);
+		if (MdmTransactionContext.OperationType.MERGE_GOLDEN_RESOURCES == theMdmTransactionContext.getRestOperation()) {
+			TerserUtil.mergeFieldsExceptIdAndMeta(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource);
+		} else {
+			TerserUtil.overwriteFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.EXCLUDE_IDS_AND_META);
+		}
 	}
 }

hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmGoldenResourceMergerSvcTest.java

@@ -161,8 +161,10 @@ public class MdmGoldenResourceMergerSvcTest extends BaseMdmR4Test {
 	public void emptyFromFullTo() {
 		myFromGoldenPatient.getName().add(new HumanName().addGiven(BAD_GIVEN_NAME));
 		populatePatient(myToGoldenPatient);
+		print(myFromGoldenPatient);
 
 		Patient mergedSourcePatient = mergeGoldenPatients();
+		print(mergedSourcePatient);
 		HumanName returnedName = mergedSourcePatient.getNameFirstRep();
 		assertEquals(GIVEN_NAME, returnedName.getGivenAsSingleString());
 		assertEquals(FAMILY_NAME, returnedName.getFamily());
@@ -393,20 +395,19 @@ public class MdmGoldenResourceMergerSvcTest extends BaseMdmR4Test {
 
 	@Test
 	public void testMergeNamesAllSame() {
-		// TODO NG - Revisit when rules are available
+		myFromGoldenPatient.addName().addGiven("Jim");
-//		myFromSourcePatient.addName().addGiven("Jim");
+		myFromGoldenPatient.getNameFirstRep().addGiven("George");
-//		myFromSourcePatient.getNameFirstRep().addGiven("George");
+		assertThat(myFromGoldenPatient.getName(), hasSize(1));
-//		assertThat(myFromSourcePatient.getName(), hasSize(1));
+		assertThat(myFromGoldenPatient.getName().get(0).getGiven(), hasSize(2));
-//		assertThat(myFromSourcePatient.getName().get(0).getGiven(), hasSize(2));
+
-//
+		myToGoldenPatient.addName().addGiven("Jim");
-//		myToSourcePatient.addName().addGiven("Jim");
+		myToGoldenPatient.getNameFirstRep().addGiven("George");
-//		myToSourcePatient.getNameFirstRep().addGiven("George");
+		assertThat(myToGoldenPatient.getName(), hasSize(1));
-//		assertThat(myToSourcePatient.getName(), hasSize(1));
+		assertThat(myToGoldenPatient.getName().get(0).getGiven(), hasSize(2));
-//		assertThat(myToSourcePatient.getName().get(0).getGiven(), hasSize(2));
+
-//
+		mergeGoldenPatients();
-//		mergeSourcePatients();
+		assertThat(myToGoldenPatient.getName(), hasSize(1));
-//		assertThat(myToSourcePatient.getName(), hasSize(1));
+		assertThat(myToGoldenPatient.getName().get(0).getGiven(), hasSize(2));
-//		assertThat(myToSourcePatient.getName().get(0).getGiven(), hasSize(2));
 	}
 
 	@Test
@@ -426,8 +427,6 @@ public class MdmGoldenResourceMergerSvcTest extends BaseMdmR4Test {
 	}
 
 	private MdmLink createMdmLink(Patient theSourcePatient, Patient theTargetPatient) {
-		//TODO GGG Ensure theis comment can be safely removed
-		//theSourcePatient.addLink().setTarget(new Reference(theTargetPatient));
 		return myMdmLinkDaoSvc.createOrUpdateLinkEntity(theSourcePatient, theTargetPatient, POSSIBLE_MATCH, MdmLinkSourceEnum.AUTO, createContextForCreate("Patient"));
 	}
 

hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmMatchLinkSvcTest.java

@@ -35,12 +35,7 @@ import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.NO_MATCH;
 import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.POSSIBLE_DUPLICATE;
 import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.POSSIBLE_MATCH;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.*;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.in;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -184,13 +179,13 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 		Optional<MdmLink> mdmLink = myMdmLinkDaoSvc.getMatchedLinkForSourcePid(patient.getIdElement().getIdPartAsLong());
 		Patient read = getTargetResourceFromMdmLink(mdmLink.get(), "Patient");
 
-		// TODO NG - rules haven't been determined yet revisit once implemented...
+		assertThat(read.getNameFirstRep().getFamily(), is(equalTo(patient.getNameFirstRep().getFamily())));
-//		assertThat(read.getNameFirstRep().getFamily(), is(equalTo(patient.getNameFirstRep().getFamily())));
+		assertThat(read.getNameFirstRep().getGivenAsSingleString(), is(equalTo(patient.getNameFirstRep().getGivenAsSingleString())));
-//		assertThat(read.getNameFirstRep().getGivenAsSingleString(), is(equalTo(patient.getNameFirstRep().getGivenAsSingleString())));
+		assertThat(read.getBirthDateElement().toHumanDisplay(), is(equalTo(patient.getBirthDateElement().toHumanDisplay())));
-//		assertThat(read.getBirthDateElement().toHumanDisplay(), is(equalTo(patient.getBirthDateElement().toHumanDisplay())));
+		assertThat(read.getTelecomFirstRep().getValue(), is(equalTo(patient.getTelecomFirstRep().getValue())));
-//		assertThat(read.getTelecomFirstRep().getValue(), is(equalTo(patient.getTelecomFirstRep().getValue())));
+		assertThat(read.getPhoto().size(), is(equalTo(patient.getPhoto().size())));
-//		assertThat(read.getPhoto().getData(), is(equalTo(patient.getPhotoFirstRep().getData())));
+		assertThat(read.getPhotoFirstRep().getData(), is(equalTo(patient.getPhotoFirstRep().getData())));
-//		assertThat(read.getGender(), is(equalTo(patient.getGender())));
+		assertThat(read.getGender(), is(equalTo(patient.getGender())));
 	}
 
 	@Test
@@ -246,7 +241,6 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 
 	@Test
 	public void testHavingMultipleEIDsOnIncomingPatientMatchesCorrectly() {
-
 		Patient patient1 = buildJanePatient();
 		addExternalEID(patient1, "id_1");
 		addExternalEID(patient1, "id_2");
@@ -274,7 +268,6 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 		List<MdmLink> possibleDuplicates = myMdmLinkDaoSvc.getPossibleDuplicates();
 		assertThat(possibleDuplicates, hasSize(1));
 
-
 		List<Long> duplicatePids = Stream.of(patient1, patient2)
 			.map(this::getGoldenResourceFromTargetResource)
 			.map(myIdHelperService::getPidOrNull)
@@ -480,8 +473,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 		Patient sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(janePaulPatient);
 		HumanName nameFirstRep = sourcePatientFromTarget.getNameFirstRep();
 
-		// TODO NG  attribute propagation has been removed - revisit once source survivorship rules are defined
+		assertThat(nameFirstRep.getGivenAsSingleString(), is(equalToIgnoringCase("paul")));
-		// assertThat(nameFirstRep.getGivenAsSingleString(), is(equalToIgnoringCase("paul")));
 	}
 
 	@Test
@@ -492,8 +484,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 
 		Patient sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(paul);
 
-		// TODO NG - rules haven't been determined yet revisit once implemented...
+		assertThat(sourcePatientFromTarget.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
-//		assertThat(sourcePatientFromTarget.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
 
 		Patient paul2 = buildPaulPatient();
 		paul2.setGender(Enumerations.AdministrativeGender.FEMALE);
@@ -504,7 +495,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 		//Newly matched patients aren't allowed to overwrite GoldenResource Attributes unless they are empty,
 		// so gender should still be set to male.
 		Patient paul2GoldenResource = (Patient) getGoldenResourceFromTargetResource(paul2);
-//		assertThat(paul2GoldenResource.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
+		assertThat(paul2GoldenResource.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
 	}
 
 	@Test
@@ -516,8 +507,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 		paul = createPatientAndUpdateLinks(paul);
 
 		Patient sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(paul);
-		// TODO NG - rules haven't been determined yet revisit once implemented...
+		assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(incorrectBirthdate));
-//		assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(incorrectBirthdate));
 
 		String correctBirthdate = "1990-06-28";
 		paul.getBirthDateElement().setValueAsString(correctBirthdate);
@@ -525,8 +515,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 		paul = updatePatientAndUpdateLinks(paul);
 
 		sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(paul);
-		// TODO NG - rules haven't been determined yet revisit once implemented...
+		assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(equalTo(correctBirthdate)));
-//		assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(equalTo(correctBirthdate)));
 		assertLinkCount(1);
 	}
 
@@ -613,6 +602,5 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
 		List<MdmLink> possibleDuplicates = myMdmLinkDaoSvc.getPossibleDuplicates();
 		assertThat(possibleDuplicates, hasSize(1));
 		assertThat(patient3, is(possibleDuplicateOf(patient1)));
-
 	}
 }

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java

@@ -144,6 +144,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider {
 	@Operation(name = ProviderConstants.MDM_MERGE_GOLDEN_RESOURCES)
 	public IBaseResource mergeGoldenResources(@OperationParam(name = ProviderConstants.MDM_MERGE_GR_FROM_GOLDEN_RESOURCE_ID, min = 1, max = 1, typeName = "string") IPrimitiveType<String> theFromGoldenResourceId,
 															@OperationParam(name = ProviderConstants.MDM_MERGE_GR_TO_GOLDEN_RESOURCE_ID, min = 1, max = 1, typeName = "string") IPrimitiveType<String> theToGoldenResourceId,
+															@OperationParam()
 															RequestDetails theRequestDetails) {
 		validateMergeParameters(theFromGoldenResourceId, theToGoldenResourceId);
 

hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/TerserUtil.java

@@ -29,11 +29,34 @@ import ca.uhn.fhir.util.FhirTerser;
 import org.hl7.fhir.instance.model.api.IBase;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 
+import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static ca.uhn.fhir.mdm.util.GoldenResourceHelper.FIELD_NAME_IDENTIFIER;
 
-final class TerserUtil {
+public final class TerserUtil {
+
+	public static final Collection<String> IDS_AND_META_EXCLUDES =
+		Collections.unmodifiableSet(Stream.of("id", "meta", "identifier").collect(Collectors.toSet()));
+
+	public static final Predicate<String> EXCLUDE_IDS_AND_META = new Predicate<String>() {
+		@Override
+		public boolean test(String s) {
+			return !IDS_AND_META_EXCLUDES.contains(s);
+		}
+	};
+
+	public static final Predicate<String> INCLUDE_ALL = new Predicate<String>() {
+		@Override
+		public boolean test(String s) {
+			return true;
+		}
+	};
 
 	private TerserUtil() {
 	}
@@ -41,9 +64,9 @@ final class TerserUtil {
 	/**
 	 * Clones the specified canonical EID into the identifier field on the resource
 	 *
-	 * @param theFhirContext Context to pull resource definitions from
+	 * @param theFhirContext         Context to pull resource definitions from
 	 * @param theResourceToCloneInto Resource to set the EID on
-	 * @param theEid EID to be set
+	 * @param theEid                 EID to be set
 	 */
 	public static void cloneEidIntoResource(FhirContext theFhirContext, IBaseResource theResourceToCloneInto, CanonicalEID theEid) {
 		// get a ref to the actual ID Field
@@ -84,7 +107,7 @@ final class TerserUtil {
 		List<IBase> theToFieldValues = childDefinition.getAccessor().getValues(theTo);
 
 		for (IBase theFromFieldValue : theFromFieldValues) {
-			if (contains(theFromFieldValue, theToFieldValues)) {
+			if (containsPrimitiveValue(theFromFieldValue, theToFieldValues)) {
 				continue;
 			}
 
@@ -95,11 +118,87 @@ final class TerserUtil {
 		}
 	}
 
-	private static boolean contains(IBase theItem, List<IBase> theItems) {
+	private static boolean containsPrimitiveValue(IBase theItem, List<IBase> theItems) {
 		PrimitiveTypeEqualsPredicate predicate = new PrimitiveTypeEqualsPredicate();
 		return theItems.stream().anyMatch(i -> {
 			return predicate.test(i, theItem);
 		});
 	}
 
+	private static boolean contains(IBase theItem, List<IBase> theItems) {
+		Method method = null;
+		for (Method m : theItem.getClass().getDeclaredMethods()) {
+			if (m.getName().equals("equalsDeep")) {
+				method = m;
+				break;
+			}
+		}
+
+		final Method m = method;
+		return theItems.stream().anyMatch(i -> {
+			if (m != null) {
+				try {
+					return (Boolean) m.invoke(theItem, i);
+				} catch (Exception e) {
+					throw new RuntimeException("Unable to compare equality via equalsDeep", e);
+				}
+			}
+			return theItem.equals(i);
+		});
+	}
+
+	public static void mergeAllFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo) {
+		mergeFields(theFhirContext, theFrom, theTo, INCLUDE_ALL);
+	}
+
+	public static void overwriteFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo, Predicate<String> inclusionStrategy) {
+		FhirTerser terser = theFhirContext.newTerser();
+
+		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
+		for (BaseRuntimeChildDefinition childDefinition : definition.getChildrenAndExtension()) {
+			if (!inclusionStrategy.test(childDefinition.getElementName())) {
+				continue;
+			}
+
+			childDefinition.getAccessor().getFirstValueOrNull(theFrom).ifPresent( v -> {
+					childDefinition.getMutator().setValue(theTo, v);
+				}
+			);
+		}
+	}
+
+	public static void mergeFieldsExceptIdAndMeta(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo) {
+		mergeFields(theFhirContext, theFrom, theTo, EXCLUDE_IDS_AND_META);
+	}
+
+	public static void mergeFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo, Predicate<String> inclusionStrategy) {
+		FhirTerser terser = theFhirContext.newTerser();
+
+		RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
+		for (BaseRuntimeChildDefinition childDefinition : definition.getChildrenAndExtension()) {
+			if (!inclusionStrategy.test(childDefinition.getElementName())) {
+				continue;
+			}
+
+			List<IBase> theFromFieldValues = childDefinition.getAccessor().getValues(theFrom);
+			List<IBase> theToFieldValues = childDefinition.getAccessor().getValues(theTo);
+			
+			for (IBase theFromFieldValue : theFromFieldValues) {
+				if (contains(theFromFieldValue, theToFieldValues)) {
+					continue;
+				}
+
+				IBase newFieldValue = childDefinition.getChildByName(childDefinition.getElementName()).newInstance();
+				terser.cloneInto(theFromFieldValue, newFieldValue, true);
+
+				try {
+					theToFieldValues.add(newFieldValue);
+				} catch (UnsupportedOperationException e) {
+					childDefinition.getMutator().setValue(theTo, newFieldValue);
+					break;
+				}
+			}
+		}
+	}
+
 }

hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/util/TerserUtilTest.java

@@ -8,7 +8,9 @@ import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.hasSize;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 class TerserUtilTest extends BaseR4Test {
 
@@ -28,12 +30,12 @@ class TerserUtilTest extends BaseR4Test {
 		assertEquals(p1.getIdentifier().get(0).getValue(), p2.getIdentifier().get(0).getValue());
 	}
 
-//	@Test
+	@Test
 	void testCloneFields() {
 		Patient p1 = buildJohny();
 		Patient p2 = new Patient();
 
-		// TerserUtil.cloneFields(ourFhirContext, p1, p2);
+		TerserUtil.mergeFieldsExceptIdAndMeta(ourFhirContext, p1, p2);
 
 		assertTrue(p2.getIdentifier().isEmpty());
 
@@ -42,8 +44,8 @@ class TerserUtilTest extends BaseR4Test {
 		assertEquals(p1.getName().get(0).getNameAsSingleString(), p2.getName().get(0).getNameAsSingleString());
 	}
 
-//	@Test
+	@Test
-	void testAnotherCloneFields() {
+	void testCloneWithNonPrimitves() {
 		Patient p1 = new Patient();
 		Patient p2 = new Patient();
 
@@ -57,9 +59,32 @@ class TerserUtilTest extends BaseR4Test {
 		assertThat(p2.getName(), hasSize(1));
 		assertThat(p2.getName().get(0).getGiven(), hasSize(2));
 
-		// TerserUtil.cloneFields(ourFhirContext, p1, p2);
+		TerserUtil.mergeAllFields(ourFhirContext, p1, p2);
 		assertThat(p2.getName(), hasSize(2));
 		assertThat(p2.getName().get(0).getGiven(), hasSize(2));
 		assertThat(p2.getName().get(1).getGiven(), hasSize(2));
 	}
+
+	@Test
+	void testCloneWithDuplicateNonPrimitives() {
+		Patient p1 = new Patient();
+		Patient p2 = new Patient();
+
+		p1.addName().addGiven("Jim");
+		p1.getNameFirstRep().addGiven("George");
+
+		assertThat(p1.getName(), hasSize(1));
+		assertThat(p1.getName().get(0).getGiven(), hasSize(2));
+
+		p2.addName().addGiven("Jim");
+		p2.getNameFirstRep().addGiven("George");
+
+		assertThat(p2.getName(), hasSize(1));
+		assertThat(p2.getName().get(0).getGiven(), hasSize(2));
+
+		TerserUtil.mergeAllFields(ourFhirContext, p1, p2);
+
+		assertThat(p2.getName(), hasSize(1));
+		assertThat(p2.getName().get(0).getGiven(), hasSize(2));
+	}
 }