3792 providing capability to alter source resource content before mdm matching (#3802)

* Solution baseline.

* Adding integration test.

* Parameterizing the test

* Adding to documentation.

* Modification following code review.

* Modification following code review.

* Adding documentation.

* Adding documentation specific to MDM customization with pointcut.

* Modifications following second code review.

* Code review modifications.

Co-authored-by: peartree <etienne.poirier@smilecdr.com>

hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java

@@ -2008,6 +2008,23 @@ public enum Pointcut implements IPointcut {
 		"ca.uhn.fhir.validation.ValidationResult"
 	),
 
+	/**
+	 * <b>MDM(EMPI) Hook:</b>
+	 * Invoked when a persisted resource (a resource that has just been stored in the
+	 * database via a create/update/patch/etc.) enters the MDM module. The purpose of the pointcut is to permit a pseudo
+	 * modification of the resource elements to influence the MDM linking process.  Any modifications to the resource are not persisted.
+	 * <p>
+	 * Hooks may accept the following parameters:
+	 * <ul>
+	 * <li>org.hl7.fhir.instance.model.api.IBaseResource - </li>
+	 * </ul>
+	 * </p>
+	 * <p>
+	 * Hooks should return <code>void</code>.
+	 * </p>
+	 */
+	MDM_BEFORE_PERSISTED_RESOURCE_CHECKED(void.class,
+		"org.hl7.fhir.instance.model.api.IBaseResource"),
 
 	/**
 	 * <b>MDM(EMPI) Hook:</b>

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3792-providing-mdm-preprocessing-pointcut.yaml

@@ -0,0 +1,4 @@
+---
+type: add
+issue: 3792
+title: "Adding pointcut `MDM_BEFORE_PERSISTED_RESOURCE_CHECKED` allowing for the customization of source resource before MDM processing."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/files.properties

@@ -72,6 +72,7 @@ page.server_jpa_mdm.mdm_eid=MDM Enterprise Identifiers
 page.server_jpa_mdm.mdm_operations=MDM Operations
 page.server_jpa_mdm.mdm_details=MDM Technical Details
 page.server_jpa_mdm.mdm_expansion=MDM Search Expansion
+page.server_jpa_mdm.mdm_customizations=MDM Customizations
 
 section.server_jpa_cql.title=JPA Server: CQL
 page.server_jpa_cql.cql=CQL Getting Started

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_customizations.md

@@ -0,0 +1,23 @@
+# MDM Customizations
+
+This section describes customization supported by MDM.
+
+# Interceptors
+
+MDM allows customization through interceptors.  Please refer to the [Interceptors](/hapi-fhir/docs/interceptors/interceptors.html) section of the documentation for further details and implementation guidelines.   
+
+## MDM Preprocessing Pointcut
+
+MDM supports a pointcut invocation right before it starts matching an incoming source resource against defined rules.  A possible use of the pointcut would be to alter a resource content with the intention of influencing the MDM matching and linking process.  Any modifications to the source resource are NOT persisted to the database.  Modifications performed within the pointcut will remain valid during MDM processing only.
+
+## Example: Ignoring Matches on Patient Name
+
+In a scenario where a patient was given a placeholder name(John Doe), it would be desirable to ignore a 'name' matching rule but allow matching on other valid rules like matching SSN or a matching address.  
+
+The following provides a full implementation of an interceptor that prevents matching on a patient name when it detects  a placeholder value.  
+
+```java
+{{snippet:file:hapi-fhir/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/PatientNameModifierMdmPreProcessingInterceptor.java}}
+```
+
+See the [Pointcut](/apidocs/hapi-fhir-base/ca/uhn/fhir/interceptor/api/Pointcut.html) JavaDoc for further details on the pointcut MDM_BEFORE_PERSISTED_RESOURCE_CHECKED.

hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java

@@ -81,8 +81,11 @@ public class MdmMessageHandler implements MessageHandler {
 
 		ResourceModifiedMessage msg = ((ResourceModifiedJsonMessage) theMessage).getPayload();
 		try {
-			if (myMdmResourceFilteringSvc.shouldBeProcessed(getResourceFromPayload(msg))) {
+
-				matchMdmAndUpdateLinks(msg);
+			IBaseResource sourceResource = msg.getNewPayload(myFhirContext);
+
+			if (myMdmResourceFilteringSvc.shouldBeProcessed((IAnyResource) sourceResource)) {
+				matchMdmAndUpdateLinks(sourceResource, msg);
 			}
 		} catch (TooManyCandidatesException e) {
 			ourLog.error(e.getMessage(), e);
@@ -93,18 +96,27 @@ public class MdmMessageHandler implements MessageHandler {
 		}
 	}
 
-	private void matchMdmAndUpdateLinks(ResourceModifiedMessage theMsg) {
+	private void matchMdmAndUpdateLinks(IBaseResource theSourceResource, ResourceModifiedMessage theMsg) {
-		String resourceType = theMsg.getPayloadId(myFhirContext).getResourceType();
+
+		String resourceType = theSourceResource.getIdElement().getResourceType();
 		validateResourceType(resourceType);
+
+		if (myInterceptorBroadcaster.hasHooks(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED)){
+			HookParams params = new HookParams().add(IBaseResource.class, theSourceResource);
+			myInterceptorBroadcaster.callHooks(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED, params);
+		}
+
+		theSourceResource.setUserData(Constants.RESOURCE_PARTITION_ID, theMsg.getPartitionId());
+
 		MdmTransactionContext mdmContext = createMdmContext(theMsg, resourceType);
 		try {
 			switch (theMsg.getOperationType()) {
 				case CREATE:
-					handleCreateResource(theMsg, mdmContext);
+					handleCreateResource(theSourceResource, mdmContext);
 					break;
 				case UPDATE:
 				case MANUALLY_TRIGGERED:
-					handleUpdateResource(theMsg, mdmContext);
+					handleUpdateResource(theSourceResource, mdmContext);
 					break;
 				case DELETE:
 				default:
@@ -115,21 +127,10 @@ public class MdmMessageHandler implements MessageHandler {
 			mdmContext.addTransactionLogMessage(e.getMessage());
 		} finally {
 			// Interceptor call: MDM_AFTER_PERSISTED_RESOURCE_CHECKED
-			IBaseResource targetResource = theMsg.getPayload(myFhirContext);
-			ResourceOperationMessage outgoingMsg = new ResourceOperationMessage(myFhirContext, targetResource, theMsg.getOperationType());
-			outgoingMsg.setTransactionId(theMsg.getTransactionId());
-
-			MdmLinkEvent linkChangeEvent = new MdmLinkEvent();
-			mdmContext.getMdmLinks()
-				.stream()
-				.forEach(l -> {
-					linkChangeEvent.addMdmLink(myModelConverter.toJson((MdmLink) l));
-				});
-
 			HookParams params = new HookParams()
-				.add(ResourceOperationMessage.class, outgoingMsg)
+				.add(ResourceOperationMessage.class, getOutgoingMessage(theMsg))
 				.add(TransactionLogMessages.class, mdmContext.getTransactionLogMessages())
-				.add(MdmLinkEvent.class, linkChangeEvent);
+				.add(MdmLinkEvent.class, buildLinkChangeEvent(mdmContext));
 
 			myInterceptorBroadcaster.callHooks(Pointcut.MDM_AFTER_PERSISTED_RESOURCE_CHECKED, params);
 		}
@@ -162,27 +163,36 @@ public class MdmMessageHandler implements MessageHandler {
 		}
 	}
 
-	private void handleCreateResource(ResourceModifiedMessage theMsg, MdmTransactionContext theMdmTransactionContext) {
+	private void handleCreateResource(IBaseResource theResource, MdmTransactionContext theMdmTransactionContext) {
-		myMdmMatchLinkSvc.updateMdmLinksForMdmSource(getResourceFromPayload(theMsg), theMdmTransactionContext);
+		myMdmMatchLinkSvc.updateMdmLinksForMdmSource((IAnyResource)theResource, theMdmTransactionContext);
 	}
 
-	private IAnyResource getResourceFromPayload(ResourceModifiedMessage theMsg) {
+	private void handleUpdateResource(IBaseResource theResource, MdmTransactionContext theMdmTransactionContext) {
-		IBaseResource newPayload = theMsg.getNewPayload(myFhirContext);
+		myMdmMatchLinkSvc.updateMdmLinksForMdmSource((IAnyResource)theResource, theMdmTransactionContext);
-		newPayload.setUserData(Constants.RESOURCE_PARTITION_ID, theMsg.getPartitionId());
-		return (IAnyResource) newPayload;
-	}
-
-	private void handleUpdateResource(ResourceModifiedMessage theMsg, MdmTransactionContext theMdmTransactionContext) {
-		myMdmMatchLinkSvc.updateMdmLinksForMdmSource(getResourceFromPayload(theMsg), theMdmTransactionContext);
-	}
-
-	private void log(MdmTransactionContext theMdmContext, String theMessage) {
-		theMdmContext.addTransactionLogMessage(theMessage);
-		ourLog.debug(theMessage);
 	}
 
 	private void log(MdmTransactionContext theMdmContext, String theMessage, Exception theException) {
 		theMdmContext.addTransactionLogMessage(theMessage);
 		ourLog.error(theMessage, theException);
 	}
+
+	private MdmLinkEvent buildLinkChangeEvent(MdmTransactionContext theMdmContext) {
+		MdmLinkEvent linkChangeEvent = new MdmLinkEvent();
+		theMdmContext.getMdmLinks()
+			.stream()
+			.forEach(l -> {
+				linkChangeEvent.addMdmLink(myModelConverter.toJson((MdmLink) l));
+			});
+
+		return linkChangeEvent;
+	}
+
+	private ResourceOperationMessage getOutgoingMessage(ResourceModifiedMessage theMsg) {
+		IBaseResource targetResource = theMsg.getPayload(myFhirContext);
+		ResourceOperationMessage outgoingMsg = new ResourceOperationMessage(myFhirContext, targetResource, theMsg.getOperationType());
+		outgoingMsg.setTransactionId(theMsg.getTransactionId());
+
+		return outgoingMsg;
+	}
+
 }

hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmPreProcessingInterceptorIT.java

@@ -0,0 +1,82 @@
+package ca.uhn.fhir.jpa.mdm.interceptor;
+
+import ca.uhn.fhir.interceptor.api.Hook;
+import ca.uhn.fhir.interceptor.api.IInterceptorService;
+import ca.uhn.fhir.interceptor.api.Pointcut;
+import ca.uhn.fhir.jpa.mdm.BaseMdmR4Test;
+import ca.uhn.fhir.jpa.mdm.helper.MdmHelperConfig;
+import ca.uhn.fhir.jpa.mdm.helper.MdmHelperR4;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.r4.model.Patient;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@ContextConfiguration(classes = {MdmHelperConfig.class})
+public class MdmPreProcessingInterceptorIT extends BaseMdmR4Test{
+
+	@RegisterExtension
+	@Autowired
+	public MdmHelperR4 myMdmHelper;
+
+	@Autowired
+	private IInterceptorService myInterceptorService;
+
+	private PatientNameModifierMdmPreProcessingInterceptor myPreProcessingInterceptor = new PatientNameModifierMdmPreProcessingInterceptor();
+	private PatientInterceptorWrapper myPatientInterceptorWrapper;
+	@BeforeEach
+	public void beforeEach(){
+		// we wrap the preProcessing interceptor to catch the return value;
+		myPatientInterceptorWrapper = new PatientInterceptorWrapper(myPreProcessingInterceptor);
+		myInterceptorService.registerInterceptor(myPatientInterceptorWrapper);
+	}
+
+	@AfterEach
+	public void afterEach(){
+		myInterceptorService.unregisterInterceptor(myPatientInterceptorWrapper);
+	}
+
+	@Test
+	public void whenInterceptorIsRegisteredThenInterceptorIsCalled() throws InterruptedException {
+
+		Patient aPatient = buildPatientWithNameAndId(NAME_GIVEN_JANE, JANE_ID);
+
+		myMdmHelper.createWithLatch(aPatient);
+
+		Patient interceptedResource = (Patient) myPatientInterceptorWrapper.getReturnedValue();
+
+		assertEquals(0, interceptedResource.getName().size());
+
+	}
+
+	public static class PatientInterceptorWrapper {
+
+		private PatientNameModifierMdmPreProcessingInterceptor myPatientInterceptor;
+
+		private IBaseResource myReturnedValue;
+
+		public PatientInterceptorWrapper(PatientNameModifierMdmPreProcessingInterceptor thePatientInterceptor) {
+			myPatientInterceptor = thePatientInterceptor;
+		}
+
+		@Hook(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED)
+		public void invoke(IBaseResource theResource) {
+			myPatientInterceptor.invoke(theResource);
+			myReturnedValue = theResource;
+		}
+
+		public IBaseResource getReturnedValue() {
+			return myReturnedValue;
+		}
+	}
+
+}
+
+
+
+

hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/PatientNameModifierMdmPreProcessingInterceptor.java

@@ -0,0 +1,30 @@
+package ca.uhn.fhir.jpa.mdm.interceptor;
+
+import ca.uhn.fhir.interceptor.api.Hook;
+import ca.uhn.fhir.interceptor.api.Pointcut;
+import org.hl7.fhir.instance.model.api.IBaseResource;
+import org.hl7.fhir.r4.model.HumanName;
+import org.hl7.fhir.r4.model.Patient;
+
+import java.util.List;
+
+import static java.util.Arrays.asList;
+
+public class PatientNameModifierMdmPreProcessingInterceptor {
+
+	List<String> myNamesToIgnore = asList("John Doe", "Jane Doe");
+
+	@Hook(Pointcut.MDM_BEFORE_PERSISTED_RESOURCE_CHECKED)
+	public void invoke(IBaseResource theResource) {
+
+		Patient patient = (Patient) theResource;
+		List<HumanName> nameList = patient.getName();
+
+		List<HumanName> validHumanNameList = nameList.stream()
+			.filter(theHumanName -> !myNamesToIgnore.contains(theHumanName.getNameAsSingleString()))
+			.toList();
+
+		patient.setName(validHumanNameList);
+	}
+
+}