diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_rules.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_rules.md index aa1320cd352..3619be3ca12 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_rules.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_rules.md @@ -51,7 +51,7 @@ Here is an example of a full HAPI MDM rules json document: { "name": "firstname-meta", "resourceType": "Patient", - "resourcePath": "name.given", + "fhirPath": "name.given.first()", "matcher": { "algorithm": "METAPHONE" } @@ -173,8 +173,8 @@ Here is a matcher matchField that uses the SOUNDEX matcher to determine whether ```json { - "name": "familyname-soundex", - "resourceType": "*", + "name": "familyname-soundex", + "resourceType": "*", "resourcePath": "name.family", "matcher": { "algorithm": "SOUNDEX" @@ -196,6 +196,53 @@ Here is a matcher matchField that only matches when two family names are identic } ``` +While it is often suitable to use the `resourcePath` field to indicate the location of the data to be matched, occasionally you will need more direct control over precisely which fields are matched. When performing string matching, the matcher will indiscriminately try to match all elements of the left resource to all elements of the right resource. For example, consider the following two patients and matcher. + +```json +{ + "resourceType": "Patient", + "name": [{ + "given": ["Frank", "John"] + }] +} +``` + +```json +{ + "resourceType": "Patient", + "name": [{ + "given": ["John", "Frank"] + }] +} +``` + +```json +{ + "name": "firstname-meta", + "resourceType": "Patient", + "resourcePath": "name.given", + "matcher": { + "algorithm": "METAPHONE" + } +} +``` + +In this example, these two patients would match, as the matcher will compare all elements of `["John", "Frank"]` to all elements of `["Frank", "John"]` and find that there are matches. This is when you would want to use a FHIRPath matcher, as FHIRPath expressions give you more direct control. This following example shows a matcher that would cause these two patient's not to match to each other. + +```json +{ + "name": "firstname-meta-fhirpath", + "resourceType": "Patient", + "fhirPath": "name.given[0]", + "matcher": { + "algorithm": "METAPHONE" + } +} +``` +Since FHIRPath expressions support indexing it is possible to directly indicate that you would only like to compare the first element of each resource. + + + Special identifier matching is also available if you need to match on a particular identifier system: ```json { diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidator.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidator.java index 5d67cb68448..959058cadbd 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidator.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidator.java @@ -23,6 +23,7 @@ package ca.uhn.fhir.mdm.rules.config; import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.RuntimeResourceDefinition; +import ca.uhn.fhir.fhirpath.IFhirPath; import ca.uhn.fhir.mdm.api.MdmConstants; import ca.uhn.fhir.mdm.api.IMdmRuleValidator; import ca.uhn.fhir.mdm.rules.json.MdmFieldMatchJson; @@ -33,7 +34,9 @@ import ca.uhn.fhir.mdm.rules.json.MdmSimilarityJson; import ca.uhn.fhir.parser.DataFormatException; import ca.uhn.fhir.rest.server.util.ISearchParamRetriever; import ca.uhn.fhir.util.FhirTerser; +import org.hl7.fhir.instance.model.api.IBase; import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.Patient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -51,16 +54,14 @@ public class MdmRuleValidator implements IMdmRuleValidator { private final FhirContext myFhirContext; private final ISearchParamRetriever mySearchParamRetriever; - private final Class myPatientClass; - private final Class myPractitionerClass; private final FhirTerser myTerser; + private final IFhirPath myFhirPath; @Autowired public MdmRuleValidator(FhirContext theFhirContext, ISearchParamRetriever theSearchParamRetriever) { myFhirContext = theFhirContext; - myPatientClass = theFhirContext.getResourceDefinition("Patient").getImplementingClass(); - myPractitionerClass = theFhirContext.getResourceDefinition("Practitioner").getImplementingClass(); myTerser = myFhirContext.newTerser(); + myFhirPath = myFhirContext.newFhirPath(); mySearchParamRetriever = theSearchParamRetriever; } @@ -158,20 +159,44 @@ public class MdmRuleValidator implements IMdmRuleValidator { } private void validateFieldPathForType(String theResourceType, MdmFieldMatchJson theFieldMatch) { - ourLog.debug(" validating resource {} for {} ", theResourceType, theFieldMatch.getResourcePath()); + ourLog.debug("Validating resource {} for {} ", theResourceType, theFieldMatch.getResourcePath()); - try { - RuntimeResourceDefinition resourceDefinition = myFhirContext.getResourceDefinition(theResourceType); - Class implementingClass = resourceDefinition.getImplementingClass(); - String path = theResourceType + "." + theFieldMatch.getResourcePath(); - myTerser.getDefinition(implementingClass, path); - } catch (DataFormatException | ConfigurationException | ClassCastException e) { - throw new ConfigurationException("MatchField " + + if (theFieldMatch.getFhirPath() != null && theFieldMatch.getResourcePath() != null) { + throw new ConfigurationException("MatchField [" + theFieldMatch.getName() + - " resourceType " + + "] resourceType [" + theFieldMatch.getResourceType() + - " has invalid path '" + theFieldMatch.getResourcePath() + "'. " + - e.getMessage()); + "] has defined both a resourcePath and a fhirPath. You must define one of the two."); + } + + if (theFieldMatch.getResourcePath() == null && theFieldMatch.getFhirPath() == null) { + throw new ConfigurationException("MatchField [" + + theFieldMatch.getName() + + "] resourceType [" + + theFieldMatch.getResourceType() + + "] has defined neither a resourcePath or a fhirPath. You must define one of the two."); + } + + if (theFieldMatch.getResourcePath() != null) { + try { //Try to validate the struture definition path + RuntimeResourceDefinition resourceDefinition = myFhirContext.getResourceDefinition(theResourceType); + Class implementingClass = resourceDefinition.getImplementingClass(); + String path = theResourceType + "." + theFieldMatch.getResourcePath(); + myTerser.getDefinition(implementingClass, path); + } catch (DataFormatException | ConfigurationException | ClassCastException e) { + //Fallback to attempting to FHIRPath evaluate it. + throw new ConfigurationException("MatchField " + + theFieldMatch.getName() + + " resourceType " + + theFieldMatch.getResourceType() + + " has invalid path '" + theFieldMatch.getResourcePath() + "'. " + e.getMessage()); + } + } else { //Try to validate the FHIRPath + try { + myFhirPath.parse(theResourceType + "." + theFieldMatch.getFhirPath()); + } catch (Exception e) { + throw new ConfigurationException("MatchField [" + theFieldMatch.getName() + "] resourceType [" + theFieldMatch.getResourceType() + "] has failed FHIRPath evaluation. " + e.getMessage()); + } } } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/json/MdmFieldMatchJson.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/json/MdmFieldMatchJson.java index aba025c0bcd..ac7f331e080 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/json/MdmFieldMatchJson.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/json/MdmFieldMatchJson.java @@ -44,9 +44,12 @@ public class MdmFieldMatchJson implements IModelJson { @JsonProperty(value = "resourceType", required = true) String myResourceType; - @JsonProperty(value = "resourcePath", required = true) + @JsonProperty(value = "resourcePath", required = false) String myResourcePath; + @JsonProperty(value = "fhirPath", required = false) + String myFhirPath; + @JsonProperty(value = "matcher", required = false) MdmMatcherJson myMatcher; @@ -112,4 +115,13 @@ public class MdmFieldMatchJson implements IModelJson { } throw new InternalErrorException("Field Match " + myName + " has neither a matcher nor a similarity."); } + + public String getFhirPath() { + return myFhirPath; + } + + public MdmFieldMatchJson setFhirPath(String theFhirPath) { + myFhirPath = theFhirPath; + return this; + } } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcher.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcher.java index 2c8807b907e..121d49c4c03 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcher.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcher.java @@ -21,6 +21,7 @@ package ca.uhn.fhir.mdm.rules.svc; */ import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.fhirpath.IFhirPath; import ca.uhn.fhir.mdm.api.MdmMatchEvaluation; import ca.uhn.fhir.mdm.rules.json.MdmFieldMatchJson; import ca.uhn.fhir.mdm.rules.json.MdmRulesJson; @@ -43,16 +44,20 @@ public class MdmResourceFieldMatcher { private final MdmFieldMatchJson myMdmFieldMatchJson; private final String myResourceType; private final String myResourcePath; + private final String myFhirPath; private final MdmRulesJson myMdmRulesJson; private final String myName; + private final boolean myIsFhirPathExpression; public MdmResourceFieldMatcher(FhirContext theFhirContext, MdmFieldMatchJson theMdmFieldMatchJson, MdmRulesJson theMdmRulesJson) { myFhirContext = theFhirContext; myMdmFieldMatchJson = theMdmFieldMatchJson; myResourceType = theMdmFieldMatchJson.getResourceType(); myResourcePath = theMdmFieldMatchJson.getResourcePath(); + myFhirPath = theMdmFieldMatchJson.getFhirPath(); myName = theMdmFieldMatchJson.getName(); myMdmRulesJson = theMdmRulesJson; + myIsFhirPathExpression = myFhirPath != null; } /** @@ -71,9 +76,18 @@ public class MdmResourceFieldMatcher { validate(theLeftResource); validate(theRightResource); - FhirTerser terser = myFhirContext.newTerser(); - List leftValues = terser.getValues(theLeftResource, myResourcePath, IBase.class); - List rightValues = terser.getValues(theRightResource, myResourcePath, IBase.class); + List leftValues; + List rightValues; + + if (myIsFhirPathExpression) { + IFhirPath fhirPath = myFhirContext.newFhirPath(); + leftValues = fhirPath.evaluate(theLeftResource, myFhirPath, IBase.class); + rightValues = fhirPath.evaluate(theRightResource, myFhirPath, IBase.class); + } else { + FhirTerser fhirTerser = myFhirContext.newTerser(); + leftValues = fhirTerser.getValues(theLeftResource, myResourcePath, IBase.class); + rightValues = fhirTerser.getValues(theRightResource, myResourcePath, IBase.class); + } return match(leftValues, rightValues); } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceMatcherSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceMatcherSvc.java index 5e8c17727b0..37bae2c95cd 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceMatcherSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceMatcherSvc.java @@ -144,7 +144,8 @@ public class MdmResourceMatcherSvc { return retVal; } - private boolean isValidResourceType(String theResourceType, String theFieldComparatorType) { + + private boolean isValidResourceType(String theResourceType, String theFieldComparatorType) { return ( theFieldComparatorType.equalsIgnoreCase(MdmConstants.ALL_RESOURCE_SEARCH_PARAM_TYPE) || theFieldComparatorType.equalsIgnoreCase(theResourceType) diff --git a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidatorTest.java b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidatorTest.java index f12ba8c35ae..514d4da0eb6 100644 --- a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidatorTest.java +++ b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/config/MdmRuleValidatorTest.java @@ -71,6 +71,36 @@ public class MdmRuleValidatorTest extends BaseR4Test { } } + @Test + public void testMatcherBadFhirPath() throws IOException { + try { + setMdmRuleJson("bad-rules-bad-fhirpath.json"); + fail(); + } catch (ConfigurationException e) { + assertThat(e.getMessage(), startsWith("MatchField [given-name] resourceType [Patient] has failed FHIRPath evaluation. Error in ?? at 1, 1: The name blurst is not a valid function name")); + } + } + + @Test + public void testBadRulesMissingBothPaths() throws IOException { + try { + setMdmRuleJson("bad-rules-no-path.json"); + fail(); + } catch (ConfigurationException e) { + assertThat(e.getMessage(), startsWith("MatchField [given-name] resourceType [Patient] has defined neither a resourcePath or a fhirPath. You must define one of the two.")); + } + } + + @Test + public void testBadRulesBothPathsFilled() throws IOException { + try { + setMdmRuleJson("bad-rules-both-paths.json"); + fail(); + } catch (ConfigurationException e) { + assertThat(e.getMessage(), startsWith("MatchField [given-name] resourceType [Patient] has defined both a resourcePath and a fhirPath. You must define one of the two.")); + } + } + @Test public void testMatcherBadSearchParam() throws IOException { try { diff --git a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/BaseMdmRulesR4Test.java b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/BaseMdmRulesR4Test.java index e7ab67d3b52..02b456673f4 100644 --- a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/BaseMdmRulesR4Test.java +++ b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/BaseMdmRulesR4Test.java @@ -16,6 +16,7 @@ import java.util.Arrays; public abstract class BaseMdmRulesR4Test extends BaseR4Test { public static final String PATIENT_GIVEN = "patient-given"; + public static final String PATIENT_GIVEN_FIRST = "patient-given-first"; public static final String PATIENT_FAMILY = "patient-last"; public static final double NAME_THRESHOLD = 0.8; @@ -36,6 +37,7 @@ public abstract class BaseMdmRulesR4Test extends BaseR4Test { .setResourceType("Patient") .setResourcePath("name.given") .setSimilarity(new MdmSimilarityJson().setAlgorithm(MdmSimilarityEnum.COSINE).setMatchThreshold(NAME_THRESHOLD)); + myBothNameFields = String.join(",", PATIENT_GIVEN, PATIENT_FAMILY); } diff --git a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcherR4Test.java b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcherR4Test.java index fea95e21926..c00841ca708 100644 --- a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcherR4Test.java +++ b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/MdmResourceFieldMatcherR4Test.java @@ -30,7 +30,6 @@ public class MdmResourceFieldMatcherR4Test extends BaseMdmRulesR4Test { @BeforeEach public void before() { super.before(); - myComparator = new MdmResourceFieldMatcher(ourFhirContext, myGivenNameMatchField, myMdmRulesJson); myJohn = buildJohn(); myJohny = buildJohny(); @@ -91,22 +90,6 @@ public class MdmResourceFieldMatcherR4Test extends BaseMdmRulesR4Test { } } - @Test - public void testBadPath() { - try { - MdmFieldMatchJson matchField = new MdmFieldMatchJson() - .setName("patient-foo") - .setResourceType("Patient") - .setResourcePath("foo") - .setSimilarity(new MdmSimilarityJson().setAlgorithm(MdmSimilarityEnum.COSINE).setMatchThreshold(NAME_THRESHOLD)); - MdmResourceFieldMatcher comparator = new MdmResourceFieldMatcher(ourFhirContext, matchField, myMdmRulesJson); - comparator.match(myJohn, myJohny); - fail(); - } catch (DataFormatException e) { - assertThat(e.getMessage(), startsWith("Unknown child name 'foo' in element Patient")); - } - } - @Test public void testMatch() { assertTrue(myComparator.match(myJohn, myJohny).match); diff --git a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/ResourceMatcherR4Test.java b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/ResourceMatcherR4Test.java index 7301b2bcbbd..0b105cf1892 100644 --- a/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/ResourceMatcherR4Test.java +++ b/hapi-fhir-server-mdm/src/test/java/ca/uhn/fhir/mdm/rules/svc/ResourceMatcherR4Test.java @@ -7,12 +7,15 @@ import ca.uhn.fhir.mdm.rules.json.MdmFieldMatchJson; import ca.uhn.fhir.mdm.rules.json.MdmMatcherJson; import ca.uhn.fhir.mdm.rules.json.MdmRulesJson; import ca.uhn.fhir.mdm.rules.matcher.MdmMatcherEnum; +import ca.uhn.fhir.util.StopWatch; import org.hl7.fhir.r4.model.HumanName; import org.hl7.fhir.r4.model.Patient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -56,8 +59,8 @@ public class ResourceMatcherR4Test extends BaseMdmRulesR4Test { @Test public void testMetaphoneMatchResult() { MdmResourceMatcherSvc matcherSvc = buildMatcher(buildNamePhoneRules(MdmMatcherEnum.METAPHONE)); - MdmMatchOutcome result = matcherSvc.match(myLeft, myRight); - assertMatchResult(MdmMatchResultEnum.MATCH, 7L, 3.0, false, false, result); + MdmMatchOutcome result = matcherSvc.match(myLeft, myRight); + assertMatchResult(MdmMatchResultEnum.MATCH, 7L, 3.0, false, false, result); } @Test