Survivorship 4

This commit is contained in:
Nick 2021-01-07 17:36:04 -05:00
parent ebefb141f3
commit bec056cf9d
6 changed files with 201 additions and 53 deletions

View File

@ -23,7 +23,9 @@ package ca.uhn.fhir.jpa.mdm.svc;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.mdm.api.IMdmSurvivorshipService;
import ca.uhn.fhir.mdm.model.MdmTransactionContext;
import ca.uhn.fhir.mdm.util.TerserUtil;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.springframework.beans.factory.annotation.Autowired;
public class MdmSurvivorshipSvcImpl implements IMdmSurvivorshipService {
@ -31,8 +33,42 @@ public class MdmSurvivorshipSvcImpl implements IMdmSurvivorshipService {
@Autowired
private FhirContext myFhirContext;
/**
* Survivorship rules may include the following data consolidation methods:
*
* <ul>
* <li>
* Length of field - apply the field value containing most or least number of characters - e.g. longest name
* </li>
* <li>
* Date time - all the field value from the oldest or the newest recrod - e.g. use the most recent phone number
* </li>
* <li>
* Frequency - use the most or least frequent number of occurrence - e.g. most common phone number
* </li>
* <li>
* Integer - number functions (largest, sum, avg) - e.g. number of patient encounters
* </li>
* <li>
* Quality of data - best quality data - e.g. data coming from a certain system is considered trusted and overrides all other values
* </li>
* <li>
* A hybrid approach combining all methods listed above as best fits
* </li>
* </ul>
*
* @param theTargetResource Target resource to merge fields from
* @param theGoldenResource Golden resource to merge fields into
* @param theMdmTransactionContext Current transaction context
* @param <T>
*/
@Override
public <T extends IBase> void applySurvivorshipRulesToGoldenResource(T theTargetResource, T theGoldenResource, MdmTransactionContext theMdmTransactionContext) {
// TerserUtil.cloneFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource);
// TerserUtil.mergeFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.DEFAULT_EXCLUDED_FIELDS);
if (MdmTransactionContext.OperationType.MERGE_GOLDEN_RESOURCES == theMdmTransactionContext.getRestOperation()) {
TerserUtil.mergeFieldsExceptIdAndMeta(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource);
} else {
TerserUtil.overwriteFields(myFhirContext, (IBaseResource) theTargetResource, (IBaseResource) theGoldenResource, TerserUtil.EXCLUDE_IDS_AND_META);
}
}
}

View File

@ -161,8 +161,10 @@ public class MdmGoldenResourceMergerSvcTest extends BaseMdmR4Test {
public void emptyFromFullTo() {
myFromGoldenPatient.getName().add(new HumanName().addGiven(BAD_GIVEN_NAME));
populatePatient(myToGoldenPatient);
print(myFromGoldenPatient);
Patient mergedSourcePatient = mergeGoldenPatients();
print(mergedSourcePatient);
HumanName returnedName = mergedSourcePatient.getNameFirstRep();
assertEquals(GIVEN_NAME, returnedName.getGivenAsSingleString());
assertEquals(FAMILY_NAME, returnedName.getFamily());
@ -393,20 +395,19 @@ public class MdmGoldenResourceMergerSvcTest extends BaseMdmR4Test {
@Test
public void testMergeNamesAllSame() {
// TODO NG - Revisit when rules are available
// myFromSourcePatient.addName().addGiven("Jim");
// myFromSourcePatient.getNameFirstRep().addGiven("George");
// assertThat(myFromSourcePatient.getName(), hasSize(1));
// assertThat(myFromSourcePatient.getName().get(0).getGiven(), hasSize(2));
//
// myToSourcePatient.addName().addGiven("Jim");
// myToSourcePatient.getNameFirstRep().addGiven("George");
// assertThat(myToSourcePatient.getName(), hasSize(1));
// assertThat(myToSourcePatient.getName().get(0).getGiven(), hasSize(2));
//
// mergeSourcePatients();
// assertThat(myToSourcePatient.getName(), hasSize(1));
// assertThat(myToSourcePatient.getName().get(0).getGiven(), hasSize(2));
myFromGoldenPatient.addName().addGiven("Jim");
myFromGoldenPatient.getNameFirstRep().addGiven("George");
assertThat(myFromGoldenPatient.getName(), hasSize(1));
assertThat(myFromGoldenPatient.getName().get(0).getGiven(), hasSize(2));
myToGoldenPatient.addName().addGiven("Jim");
myToGoldenPatient.getNameFirstRep().addGiven("George");
assertThat(myToGoldenPatient.getName(), hasSize(1));
assertThat(myToGoldenPatient.getName().get(0).getGiven(), hasSize(2));
mergeGoldenPatients();
assertThat(myToGoldenPatient.getName(), hasSize(1));
assertThat(myToGoldenPatient.getName().get(0).getGiven(), hasSize(2));
}
@Test
@ -426,8 +427,6 @@ public class MdmGoldenResourceMergerSvcTest extends BaseMdmR4Test {
}
private MdmLink createMdmLink(Patient theSourcePatient, Patient theTargetPatient) {
//TODO GGG Ensure theis comment can be safely removed
//theSourcePatient.addLink().setTarget(new Reference(theTargetPatient));
return myMdmLinkDaoSvc.createOrUpdateLinkEntity(theSourcePatient, theTargetPatient, POSSIBLE_MATCH, MdmLinkSourceEnum.AUTO, createContextForCreate("Patient"));
}

View File

@ -35,12 +35,7 @@ import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.NO_MATCH;
import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.POSSIBLE_DUPLICATE;
import static ca.uhn.fhir.mdm.api.MdmMatchResultEnum.POSSIBLE_MATCH;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.blankOrNullString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.in;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -184,13 +179,13 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
Optional<MdmLink> mdmLink = myMdmLinkDaoSvc.getMatchedLinkForSourcePid(patient.getIdElement().getIdPartAsLong());
Patient read = getTargetResourceFromMdmLink(mdmLink.get(), "Patient");
// TODO NG - rules haven't been determined yet revisit once implemented...
// assertThat(read.getNameFirstRep().getFamily(), is(equalTo(patient.getNameFirstRep().getFamily())));
// assertThat(read.getNameFirstRep().getGivenAsSingleString(), is(equalTo(patient.getNameFirstRep().getGivenAsSingleString())));
// assertThat(read.getBirthDateElement().toHumanDisplay(), is(equalTo(patient.getBirthDateElement().toHumanDisplay())));
// assertThat(read.getTelecomFirstRep().getValue(), is(equalTo(patient.getTelecomFirstRep().getValue())));
// assertThat(read.getPhoto().getData(), is(equalTo(patient.getPhotoFirstRep().getData())));
// assertThat(read.getGender(), is(equalTo(patient.getGender())));
assertThat(read.getNameFirstRep().getFamily(), is(equalTo(patient.getNameFirstRep().getFamily())));
assertThat(read.getNameFirstRep().getGivenAsSingleString(), is(equalTo(patient.getNameFirstRep().getGivenAsSingleString())));
assertThat(read.getBirthDateElement().toHumanDisplay(), is(equalTo(patient.getBirthDateElement().toHumanDisplay())));
assertThat(read.getTelecomFirstRep().getValue(), is(equalTo(patient.getTelecomFirstRep().getValue())));
assertThat(read.getPhoto().size(), is(equalTo(patient.getPhoto().size())));
assertThat(read.getPhotoFirstRep().getData(), is(equalTo(patient.getPhotoFirstRep().getData())));
assertThat(read.getGender(), is(equalTo(patient.getGender())));
}
@Test
@ -246,7 +241,6 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
@Test
public void testHavingMultipleEIDsOnIncomingPatientMatchesCorrectly() {
Patient patient1 = buildJanePatient();
addExternalEID(patient1, "id_1");
addExternalEID(patient1, "id_2");
@ -274,7 +268,6 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
List<MdmLink> possibleDuplicates = myMdmLinkDaoSvc.getPossibleDuplicates();
assertThat(possibleDuplicates, hasSize(1));
List<Long> duplicatePids = Stream.of(patient1, patient2)
.map(this::getGoldenResourceFromTargetResource)
.map(myIdHelperService::getPidOrNull)
@ -480,8 +473,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
Patient sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(janePaulPatient);
HumanName nameFirstRep = sourcePatientFromTarget.getNameFirstRep();
// TODO NG attribute propagation has been removed - revisit once source survivorship rules are defined
// assertThat(nameFirstRep.getGivenAsSingleString(), is(equalToIgnoringCase("paul")));
assertThat(nameFirstRep.getGivenAsSingleString(), is(equalToIgnoringCase("paul")));
}
@Test
@ -492,8 +484,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
Patient sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(paul);
// TODO NG - rules haven't been determined yet revisit once implemented...
// assertThat(sourcePatientFromTarget.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
assertThat(sourcePatientFromTarget.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
Patient paul2 = buildPaulPatient();
paul2.setGender(Enumerations.AdministrativeGender.FEMALE);
@ -504,7 +495,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
//Newly matched patients aren't allowed to overwrite GoldenResource Attributes unless they are empty,
// so gender should still be set to male.
Patient paul2GoldenResource = (Patient) getGoldenResourceFromTargetResource(paul2);
// assertThat(paul2GoldenResource.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
assertThat(paul2GoldenResource.getGender(), is(equalTo(Enumerations.AdministrativeGender.MALE)));
}
@Test
@ -516,8 +507,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
paul = createPatientAndUpdateLinks(paul);
Patient sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(paul);
// TODO NG - rules haven't been determined yet revisit once implemented...
// assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(incorrectBirthdate));
assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(incorrectBirthdate));
String correctBirthdate = "1990-06-28";
paul.getBirthDateElement().setValueAsString(correctBirthdate);
@ -525,8 +515,7 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
paul = updatePatientAndUpdateLinks(paul);
sourcePatientFromTarget = (Patient) getGoldenResourceFromTargetResource(paul);
// TODO NG - rules haven't been determined yet revisit once implemented...
// assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(equalTo(correctBirthdate)));
assertThat(sourcePatientFromTarget.getBirthDateElement().getValueAsString(), is(equalTo(correctBirthdate)));
assertLinkCount(1);
}
@ -613,6 +602,5 @@ public class MdmMatchLinkSvcTest extends BaseMdmR4Test {
List<MdmLink> possibleDuplicates = myMdmLinkDaoSvc.getPossibleDuplicates();
assertThat(possibleDuplicates, hasSize(1));
assertThat(patient3, is(possibleDuplicateOf(patient1)));
}
}

View File

@ -144,6 +144,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider {
@Operation(name = ProviderConstants.MDM_MERGE_GOLDEN_RESOURCES)
public IBaseResource mergeGoldenResources(@OperationParam(name = ProviderConstants.MDM_MERGE_GR_FROM_GOLDEN_RESOURCE_ID, min = 1, max = 1, typeName = "string") IPrimitiveType<String> theFromGoldenResourceId,
@OperationParam(name = ProviderConstants.MDM_MERGE_GR_TO_GOLDEN_RESOURCE_ID, min = 1, max = 1, typeName = "string") IPrimitiveType<String> theToGoldenResourceId,
@OperationParam()
RequestDetails theRequestDetails) {
validateMergeParameters(theFromGoldenResourceId, theToGoldenResourceId);

View File

@ -29,11 +29,34 @@ import ca.uhn.fhir.util.FhirTerser;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseResource;
import java.lang.reflect.Method;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static ca.uhn.fhir.mdm.util.GoldenResourceHelper.FIELD_NAME_IDENTIFIER;
final class TerserUtil {
public final class TerserUtil {
public static final Collection<String> IDS_AND_META_EXCLUDES =
Collections.unmodifiableSet(Stream.of("id", "meta", "identifier").collect(Collectors.toSet()));
public static final Predicate<String> EXCLUDE_IDS_AND_META = new Predicate<String>() {
@Override
public boolean test(String s) {
return !IDS_AND_META_EXCLUDES.contains(s);
}
};
public static final Predicate<String> INCLUDE_ALL = new Predicate<String>() {
@Override
public boolean test(String s) {
return true;
}
};
private TerserUtil() {
}
@ -84,7 +107,7 @@ final class TerserUtil {
List<IBase> theToFieldValues = childDefinition.getAccessor().getValues(theTo);
for (IBase theFromFieldValue : theFromFieldValues) {
if (contains(theFromFieldValue, theToFieldValues)) {
if (containsPrimitiveValue(theFromFieldValue, theToFieldValues)) {
continue;
}
@ -95,11 +118,87 @@ final class TerserUtil {
}
}
private static boolean contains(IBase theItem, List<IBase> theItems) {
private static boolean containsPrimitiveValue(IBase theItem, List<IBase> theItems) {
PrimitiveTypeEqualsPredicate predicate = new PrimitiveTypeEqualsPredicate();
return theItems.stream().anyMatch(i -> {
return predicate.test(i, theItem);
});
}
private static boolean contains(IBase theItem, List<IBase> theItems) {
Method method = null;
for (Method m : theItem.getClass().getDeclaredMethods()) {
if (m.getName().equals("equalsDeep")) {
method = m;
break;
}
}
final Method m = method;
return theItems.stream().anyMatch(i -> {
if (m != null) {
try {
return (Boolean) m.invoke(theItem, i);
} catch (Exception e) {
throw new RuntimeException("Unable to compare equality via equalsDeep", e);
}
}
return theItem.equals(i);
});
}
public static void mergeAllFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo) {
mergeFields(theFhirContext, theFrom, theTo, INCLUDE_ALL);
}
public static void overwriteFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo, Predicate<String> inclusionStrategy) {
FhirTerser terser = theFhirContext.newTerser();
RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
for (BaseRuntimeChildDefinition childDefinition : definition.getChildrenAndExtension()) {
if (!inclusionStrategy.test(childDefinition.getElementName())) {
continue;
}
childDefinition.getAccessor().getFirstValueOrNull(theFrom).ifPresent( v -> {
childDefinition.getMutator().setValue(theTo, v);
}
);
}
}
public static void mergeFieldsExceptIdAndMeta(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo) {
mergeFields(theFhirContext, theFrom, theTo, EXCLUDE_IDS_AND_META);
}
public static void mergeFields(FhirContext theFhirContext, IBaseResource theFrom, IBaseResource theTo, Predicate<String> inclusionStrategy) {
FhirTerser terser = theFhirContext.newTerser();
RuntimeResourceDefinition definition = theFhirContext.getResourceDefinition(theFrom);
for (BaseRuntimeChildDefinition childDefinition : definition.getChildrenAndExtension()) {
if (!inclusionStrategy.test(childDefinition.getElementName())) {
continue;
}
List<IBase> theFromFieldValues = childDefinition.getAccessor().getValues(theFrom);
List<IBase> theToFieldValues = childDefinition.getAccessor().getValues(theTo);
for (IBase theFromFieldValue : theFromFieldValues) {
if (contains(theFromFieldValue, theToFieldValues)) {
continue;
}
IBase newFieldValue = childDefinition.getChildByName(childDefinition.getElementName()).newInstance();
terser.cloneInto(theFromFieldValue, newFieldValue, true);
try {
theToFieldValues.add(newFieldValue);
} catch (UnsupportedOperationException e) {
childDefinition.getMutator().setValue(theTo, newFieldValue);
break;
}
}
}
}
}

View File

@ -8,7 +8,9 @@ import org.junit.jupiter.api.Test;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.jupiter.api.Assertions.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class TerserUtilTest extends BaseR4Test {
@ -28,12 +30,12 @@ class TerserUtilTest extends BaseR4Test {
assertEquals(p1.getIdentifier().get(0).getValue(), p2.getIdentifier().get(0).getValue());
}
// @Test
@Test
void testCloneFields() {
Patient p1 = buildJohny();
Patient p2 = new Patient();
// TerserUtil.cloneFields(ourFhirContext, p1, p2);
TerserUtil.mergeFieldsExceptIdAndMeta(ourFhirContext, p1, p2);
assertTrue(p2.getIdentifier().isEmpty());
@ -42,8 +44,8 @@ class TerserUtilTest extends BaseR4Test {
assertEquals(p1.getName().get(0).getNameAsSingleString(), p2.getName().get(0).getNameAsSingleString());
}
// @Test
void testAnotherCloneFields() {
@Test
void testCloneWithNonPrimitves() {
Patient p1 = new Patient();
Patient p2 = new Patient();
@ -57,9 +59,32 @@ class TerserUtilTest extends BaseR4Test {
assertThat(p2.getName(), hasSize(1));
assertThat(p2.getName().get(0).getGiven(), hasSize(2));
// TerserUtil.cloneFields(ourFhirContext, p1, p2);
TerserUtil.mergeAllFields(ourFhirContext, p1, p2);
assertThat(p2.getName(), hasSize(2));
assertThat(p2.getName().get(0).getGiven(), hasSize(2));
assertThat(p2.getName().get(1).getGiven(), hasSize(2));
}
@Test
void testCloneWithDuplicateNonPrimitives() {
Patient p1 = new Patient();
Patient p2 = new Patient();
p1.addName().addGiven("Jim");
p1.getNameFirstRep().addGiven("George");
assertThat(p1.getName(), hasSize(1));
assertThat(p1.getName().get(0).getGiven(), hasSize(2));
p2.addName().addGiven("Jim");
p2.getNameFirstRep().addGiven("George");
assertThat(p2.getName(), hasSize(1));
assertThat(p2.getName().get(0).getGiven(), hasSize(2));
TerserUtil.mergeAllFields(ourFhirContext, p1, p2);
assertThat(p2.getName(), hasSize(1));
assertThat(p2.getName().get(0).getGiven(), hasSize(2));
}
}