From d405fa1f32a857a794ecf3ebcaa0fae108b1fd4c Mon Sep 17 00:00:00 2001 From: Etienne Poirier <33007955+epeartree@users.noreply.github.com> Date: Wed, 27 Jul 2022 08:45:59 -0400 Subject: [PATCH] 3792 providing capability to alter source resource content before mdm matching (#3802) * Solution baseline. * Adding integration test. * Parameterizing the test * Adding to documentation. * Modification following code review. * Modification following code review. * Adding documentation. * Adding documentation specific to MDM customization with pointcut. * Modifications following second code review. * Code review modifications. Co-authored-by: peartree --- .../ca/uhn/fhir/interceptor/api/Pointcut.java | 17 ++++ ...-providing-mdm-preprocessing-pointcut.yaml | 4 + .../ca/uhn/hapi/fhir/docs/files.properties | 1 + .../docs/server_jpa_mdm/mdm_customizations.md | 23 ++++++ .../jpa/mdm/broker/MdmMessageHandler.java | 78 ++++++++++-------- .../MdmPreProcessingInterceptorIT.java | 82 +++++++++++++++++++ ...meModifierMdmPreProcessingInterceptor.java | 30 +++++++ 7 files changed, 201 insertions(+), 34 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3792-providing-mdm-preprocessing-pointcut.yaml create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_customizations.md create mode 100644 hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmPreProcessingInterceptorIT.java create mode 100644 hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/PatientNameModifierMdmPreProcessingInterceptor.java diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java index 61ecb4fee3a..5277f949c26 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java @@ -2008,6 +2008,23 @@ public enum Pointcut implements IPointcut { "ca.uhn.fhir.validation.ValidationResult" ), + /** + * MDM(EMPI) Hook: + * Invoked when a persisted resource (a resource that has just been stored in the + * database via a create/update/patch/etc.) enters the MDM module. The purpose of the pointcut is to permit a pseudo + * modification of the resource elements to influence the MDM linking process. Any modifications to the resource are not persisted. + *

+ * Hooks may accept the following parameters: + *

+ *

+ *

+ * Hooks should return void. + *

+ */ + MDM_BEFORE_PERSISTED_RESOURCE_CHECKED(void.class, + "org.hl7.fhir.instance.model.api.IBaseResource"), /** * MDM(EMPI) Hook: diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3792-providing-mdm-preprocessing-pointcut.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3792-providing-mdm-preprocessing-pointcut.yaml new file mode 100644 index 00000000000..22e2c120536 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3792-providing-mdm-preprocessing-pointcut.yaml @@ -0,0 +1,4 @@ +--- +type: add +issue: 3792 +title: "Adding pointcut `MDM_BEFORE_PERSISTED_RESOURCE_CHECKED` allowing for the customization of source resource before MDM processing." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties index 7d27ab21c4d..fd2c3ef2f56 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties @@ -72,6 +72,7 @@ page.server_jpa_mdm.mdm_eid=MDM Enterprise Identifiers page.server_jpa_mdm.mdm_operations=MDM Operations page.server_jpa_mdm.mdm_details=MDM Technical Details page.server_jpa_mdm.mdm_expansion=MDM Search Expansion +page.server_jpa_mdm.mdm_customizations=MDM Customizations section.server_jpa_cql.title=JPA Server: CQL page.server_jpa_cql.cql=CQL Getting Started diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_customizations.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_customizations.md new file mode 100644 index 00000000000..678ba4fa01a --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_customizations.md @@ -0,0 +1,23 @@ +# MDM Customizations + +This section describes customization supported by MDM. + +# Interceptors + +MDM allows customization through interceptors. Please refer to the [Interceptors](/hapi-fhir/docs/interceptors/interceptors.html) section of the documentation for further details and implementation guidelines. + +## MDM Preprocessing Pointcut + +MDM supports a pointcut invocation right before it starts matching an incoming source resource against defined rules. A possible use of the pointcut would be to alter a resource content with the intention of influencing the MDM matching and linking process. Any modifications to the source resource are NOT persisted to the database. Modifications performed within the pointcut will remain valid during MDM processing only. + +## Example: Ignoring Matches on Patient Name + +In a scenario where a patient was given a placeholder name(John Doe), it would be desirable to ignore a 'name' matching rule but allow matching on other valid rules like matching SSN or a matching address. + +The following provides a full implementation of an interceptor that prevents matching on a patient name when it detects a placeholder value. + +```java +{{snippet:file:hapi-fhir/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/PatientNameModifierMdmPreProcessingInterceptor.java}} +``` + +See the [Pointcut](/apidocs/hapi-fhir-base/ca/uhn/fhir/interceptor/api/Pointcut.html) JavaDoc for further details on the pointcut MDM_BEFORE_PERSISTED_RESOURCE_CHECKED. diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java index e8e2b575ad5..a2282cca805 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java @@ -81,8 +81,11 @@ public class MdmMessageHandler implements MessageHandler { ResourceModifiedMessage msg = ((ResourceModifiedJsonMessage) theMessage).getPayload(); try { - if (myMdmResourceFilteringSvc.shouldBeProcessed(getResourceFromPayload(msg))) { - matchMdmAndUpdateLinks(msg); + + IBaseResource sourceResource = msg.getNewPayload(myFhirContext); + + if (myMdmResourceFilteringSvc.shouldBeProcessed((IAnyResource) sourceResource)) { + matchMdmAndUpdateLinks(sourceResource, msg); } } catch (TooManyCandidatesException e) { ourLog.error(e.getMessage(), e); @@ -93,18 +96,27 @@ public class MdmMessageHandler implements MessageHandler { } } - private void matchMdmAndUpdateLinks(ResourceModifiedMessage theMsg) { - String resourceType = theMsg.getPayloadId(myFhirContext).getResourceType(); + private void matchMdmAndUpdateLinks(IBaseResource theSourceResource, ResourceModifiedMessage theMsg) { + + String resourceType = theSourceResource.getIdElement().getResourceType(); validateResourceType(resourceType); + + if (myInterceptorBroadcaster.hasHooks(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED)){ + HookParams params = new HookParams().add(IBaseResource.class, theSourceResource); + myInterceptorBroadcaster.callHooks(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED, params); + } + + theSourceResource.setUserData(Constants.RESOURCE_PARTITION_ID, theMsg.getPartitionId()); + MdmTransactionContext mdmContext = createMdmContext(theMsg, resourceType); try { switch (theMsg.getOperationType()) { case CREATE: - handleCreateResource(theMsg, mdmContext); + handleCreateResource(theSourceResource, mdmContext); break; case UPDATE: case MANUALLY_TRIGGERED: - handleUpdateResource(theMsg, mdmContext); + handleUpdateResource(theSourceResource, mdmContext); break; case DELETE: default: @@ -115,21 +127,10 @@ public class MdmMessageHandler implements MessageHandler { mdmContext.addTransactionLogMessage(e.getMessage()); } finally { // Interceptor call: MDM_AFTER_PERSISTED_RESOURCE_CHECKED - IBaseResource targetResource = theMsg.getPayload(myFhirContext); - ResourceOperationMessage outgoingMsg = new ResourceOperationMessage(myFhirContext, targetResource, theMsg.getOperationType()); - outgoingMsg.setTransactionId(theMsg.getTransactionId()); - - MdmLinkEvent linkChangeEvent = new MdmLinkEvent(); - mdmContext.getMdmLinks() - .stream() - .forEach(l -> { - linkChangeEvent.addMdmLink(myModelConverter.toJson((MdmLink) l)); - }); - HookParams params = new HookParams() - .add(ResourceOperationMessage.class, outgoingMsg) + .add(ResourceOperationMessage.class, getOutgoingMessage(theMsg)) .add(TransactionLogMessages.class, mdmContext.getTransactionLogMessages()) - .add(MdmLinkEvent.class, linkChangeEvent); + .add(MdmLinkEvent.class, buildLinkChangeEvent(mdmContext)); myInterceptorBroadcaster.callHooks(Pointcut.MDM_AFTER_PERSISTED_RESOURCE_CHECKED, params); } @@ -162,27 +163,36 @@ public class MdmMessageHandler implements MessageHandler { } } - private void handleCreateResource(ResourceModifiedMessage theMsg, MdmTransactionContext theMdmTransactionContext) { - myMdmMatchLinkSvc.updateMdmLinksForMdmSource(getResourceFromPayload(theMsg), theMdmTransactionContext); + private void handleCreateResource(IBaseResource theResource, MdmTransactionContext theMdmTransactionContext) { + myMdmMatchLinkSvc.updateMdmLinksForMdmSource((IAnyResource)theResource, theMdmTransactionContext); } - private IAnyResource getResourceFromPayload(ResourceModifiedMessage theMsg) { - IBaseResource newPayload = theMsg.getNewPayload(myFhirContext); - newPayload.setUserData(Constants.RESOURCE_PARTITION_ID, theMsg.getPartitionId()); - return (IAnyResource) newPayload; - } - - private void handleUpdateResource(ResourceModifiedMessage theMsg, MdmTransactionContext theMdmTransactionContext) { - myMdmMatchLinkSvc.updateMdmLinksForMdmSource(getResourceFromPayload(theMsg), theMdmTransactionContext); - } - - private void log(MdmTransactionContext theMdmContext, String theMessage) { - theMdmContext.addTransactionLogMessage(theMessage); - ourLog.debug(theMessage); + private void handleUpdateResource(IBaseResource theResource, MdmTransactionContext theMdmTransactionContext) { + myMdmMatchLinkSvc.updateMdmLinksForMdmSource((IAnyResource)theResource, theMdmTransactionContext); } private void log(MdmTransactionContext theMdmContext, String theMessage, Exception theException) { theMdmContext.addTransactionLogMessage(theMessage); ourLog.error(theMessage, theException); } + + private MdmLinkEvent buildLinkChangeEvent(MdmTransactionContext theMdmContext) { + MdmLinkEvent linkChangeEvent = new MdmLinkEvent(); + theMdmContext.getMdmLinks() + .stream() + .forEach(l -> { + linkChangeEvent.addMdmLink(myModelConverter.toJson((MdmLink) l)); + }); + + return linkChangeEvent; + } + + private ResourceOperationMessage getOutgoingMessage(ResourceModifiedMessage theMsg) { + IBaseResource targetResource = theMsg.getPayload(myFhirContext); + ResourceOperationMessage outgoingMsg = new ResourceOperationMessage(myFhirContext, targetResource, theMsg.getOperationType()); + outgoingMsg.setTransactionId(theMsg.getTransactionId()); + + return outgoingMsg; + } + } diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmPreProcessingInterceptorIT.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmPreProcessingInterceptorIT.java new file mode 100644 index 00000000000..4d137df4c6b --- /dev/null +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmPreProcessingInterceptorIT.java @@ -0,0 +1,82 @@ +package ca.uhn.fhir.jpa.mdm.interceptor; + +import ca.uhn.fhir.interceptor.api.Hook; +import ca.uhn.fhir.interceptor.api.IInterceptorService; +import ca.uhn.fhir.interceptor.api.Pointcut; +import ca.uhn.fhir.jpa.mdm.BaseMdmR4Test; +import ca.uhn.fhir.jpa.mdm.helper.MdmHelperConfig; +import ca.uhn.fhir.jpa.mdm.helper.MdmHelperR4; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Patient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@ContextConfiguration(classes = {MdmHelperConfig.class}) +public class MdmPreProcessingInterceptorIT extends BaseMdmR4Test{ + + @RegisterExtension + @Autowired + public MdmHelperR4 myMdmHelper; + + @Autowired + private IInterceptorService myInterceptorService; + + private PatientNameModifierMdmPreProcessingInterceptor myPreProcessingInterceptor = new PatientNameModifierMdmPreProcessingInterceptor(); + private PatientInterceptorWrapper myPatientInterceptorWrapper; + @BeforeEach + public void beforeEach(){ + // we wrap the preProcessing interceptor to catch the return value; + myPatientInterceptorWrapper = new PatientInterceptorWrapper(myPreProcessingInterceptor); + myInterceptorService.registerInterceptor(myPatientInterceptorWrapper); + } + + @AfterEach + public void afterEach(){ + myInterceptorService.unregisterInterceptor(myPatientInterceptorWrapper); + } + + @Test + public void whenInterceptorIsRegisteredThenInterceptorIsCalled() throws InterruptedException { + + Patient aPatient = buildPatientWithNameAndId(NAME_GIVEN_JANE, JANE_ID); + + myMdmHelper.createWithLatch(aPatient); + + Patient interceptedResource = (Patient) myPatientInterceptorWrapper.getReturnedValue(); + + assertEquals(0, interceptedResource.getName().size()); + + } + + public static class PatientInterceptorWrapper { + + private PatientNameModifierMdmPreProcessingInterceptor myPatientInterceptor; + + private IBaseResource myReturnedValue; + + public PatientInterceptorWrapper(PatientNameModifierMdmPreProcessingInterceptor thePatientInterceptor) { + myPatientInterceptor = thePatientInterceptor; + } + + @Hook(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED) + public void invoke(IBaseResource theResource) { + myPatientInterceptor.invoke(theResource); + myReturnedValue = theResource; + } + + public IBaseResource getReturnedValue() { + return myReturnedValue; + } + } + +} + + + + diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/PatientNameModifierMdmPreProcessingInterceptor.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/PatientNameModifierMdmPreProcessingInterceptor.java new file mode 100644 index 00000000000..b17ed515aab --- /dev/null +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/PatientNameModifierMdmPreProcessingInterceptor.java @@ -0,0 +1,30 @@ +package ca.uhn.fhir.jpa.mdm.interceptor; + +import ca.uhn.fhir.interceptor.api.Hook; +import ca.uhn.fhir.interceptor.api.Pointcut; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.HumanName; +import org.hl7.fhir.r4.model.Patient; + +import java.util.List; + +import static java.util.Arrays.asList; + +public class PatientNameModifierMdmPreProcessingInterceptor { + + List myNamesToIgnore = asList("John Doe", "Jane Doe"); + + @Hook(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED) + public void invoke(IBaseResource theResource) { + + Patient patient = (Patient) theResource; + List nameList = patient.getName(); + + List validHumanNameList = nameList.stream() + .filter(theHumanName -> !myNamesToIgnore.contains(theHumanName.getNameAsSingleString())) + .toList(); + + patient.setName(validHumanNameList); + } + +}