diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2739-max-mdm-matches.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2739-max-mdm-matches.yaml new file mode 100644 index 00000000000..57c940f44eb --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_5_0/2739-max-mdm-matches.yaml @@ -0,0 +1,5 @@ +--- +type: fix +issue: 2739 +title: "Too many MDM candidates matching could result in an OutOfMemoryError. Candidate matching is now limited to the +value of IMdmSettings.getCandidateSearchLimit(), default 10000." diff --git a/hapi-fhir-jpaserver-cql/src/test/java/ca/uhn/fhir/cql/r4/CqlProviderR4Test.java b/hapi-fhir-jpaserver-cql/src/test/java/ca/uhn/fhir/cql/r4/CqlProviderR4Test.java index ed740d6a19c..89e15e71605 100644 --- a/hapi-fhir-jpaserver-cql/src/test/java/ca/uhn/fhir/cql/r4/CqlProviderR4Test.java +++ b/hapi-fhir-jpaserver-cql/src/test/java/ca/uhn/fhir/cql/r4/CqlProviderR4Test.java @@ -28,7 +28,6 @@ public class CqlProviderR4Test extends BaseCqlR4Test implements CqlProviderTestB private static final String patient = "Patient/Patient-6529"; private static final String periodStart = "2000-01-01"; private static final String periodEnd = "2019-12-31"; - private static final Object syncObject = new Object(); private static boolean bundlesLoaded = false; @Autowired diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java index 4f9ffdd75b6..3a84eebb083 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/broker/MdmMessageHandler.java @@ -21,16 +21,17 @@ package ca.uhn.fhir.jpa.mdm.broker; */ import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.mdm.api.IMdmSettings; -import ca.uhn.fhir.mdm.log.Logs; -import ca.uhn.fhir.mdm.model.MdmTransactionContext; import ca.uhn.fhir.interceptor.api.HookParams; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.jpa.mdm.svc.MdmMatchLinkSvc; import ca.uhn.fhir.jpa.mdm.svc.MdmResourceFilteringSvc; +import ca.uhn.fhir.jpa.mdm.svc.candidate.TooManyCandidatesException; import ca.uhn.fhir.jpa.subscription.model.ResourceModifiedJsonMessage; import ca.uhn.fhir.jpa.subscription.model.ResourceModifiedMessage; +import ca.uhn.fhir.mdm.api.IMdmSettings; +import ca.uhn.fhir.mdm.log.Logs; +import ca.uhn.fhir.mdm.model.MdmTransactionContext; import ca.uhn.fhir.rest.server.TransactionLogMessages; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.messaging.ResourceOperationMessage; @@ -72,6 +73,9 @@ public class MdmMessageHandler implements MessageHandler { if (myMdmResourceFilteringSvc.shouldBeProcessed(getResourceFromPayload(msg))) { matchMdmAndUpdateLinks(msg); } + } catch (TooManyCandidatesException e) { + ourLog.error(e.getMessage(), e); + // skip this one with an error message and continue processing } catch (Exception e) { ourLog.error("Failed to handle MDM Matching Resource:", e); throw e; @@ -97,6 +101,7 @@ public class MdmMessageHandler implements MessageHandler { } }catch (Exception e) { log(mdmContext, "Failure during MDM processing: " + e.getMessage(), e); + mdmContext.addTransactionLogMessage(e.getMessage()); } finally { // Interceptor call: MDM_AFTER_PERSISTED_RESOURCE_CHECKED diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java index f896e4c4abc..51981cdecaa 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java @@ -21,15 +21,43 @@ package ca.uhn.fhir.jpa.mdm.config; */ import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.dao.mdm.MdmLinkDeleteSvc; import ca.uhn.fhir.jpa.interceptor.MdmSearchExpandingInterceptor; +import ca.uhn.fhir.jpa.mdm.broker.MdmMessageHandler; +import ca.uhn.fhir.jpa.mdm.broker.MdmQueueConsumerLoader; +import ca.uhn.fhir.jpa.mdm.dao.MdmLinkDaoSvc; +import ca.uhn.fhir.jpa.mdm.dao.MdmLinkFactory; +import ca.uhn.fhir.jpa.mdm.interceptor.IMdmStorageInterceptor; +import ca.uhn.fhir.jpa.mdm.interceptor.MdmStorageInterceptor; +import ca.uhn.fhir.jpa.mdm.svc.GoldenResourceMergerSvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.MdmClearSvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.MdmControllerSvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.MdmEidUpdateService; +import ca.uhn.fhir.jpa.mdm.svc.MdmGoldenResourceDeletingSvc; +import ca.uhn.fhir.jpa.mdm.svc.MdmLinkQuerySvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.MdmLinkSvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.MdmLinkUpdaterSvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.MdmMatchFinderSvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.MdmMatchLinkSvc; +import ca.uhn.fhir.jpa.mdm.svc.MdmResourceDaoSvc; +import ca.uhn.fhir.jpa.mdm.svc.MdmResourceFilteringSvc; +import ca.uhn.fhir.jpa.mdm.svc.MdmSearchParamSvc; import ca.uhn.fhir.jpa.mdm.svc.MdmSurvivorshipSvcImpl; +import ca.uhn.fhir.jpa.mdm.svc.candidate.CandidateSearcher; +import ca.uhn.fhir.jpa.mdm.svc.candidate.FindCandidateByEidSvc; +import ca.uhn.fhir.jpa.mdm.svc.candidate.FindCandidateByExampleSvc; +import ca.uhn.fhir.jpa.mdm.svc.candidate.FindCandidateByLinkSvc; +import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmCandidateSearchCriteriaBuilderSvc; +import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmCandidateSearchSvc; +import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmGoldenResourceFindingSvc; +import ca.uhn.fhir.mdm.api.IGoldenResourceMergerSvc; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; import ca.uhn.fhir.mdm.api.IMdmExpungeSvc; import ca.uhn.fhir.mdm.api.IMdmLinkQuerySvc; import ca.uhn.fhir.mdm.api.IMdmLinkSvc; import ca.uhn.fhir.mdm.api.IMdmLinkUpdaterSvc; import ca.uhn.fhir.mdm.api.IMdmMatchFinderSvc; -import ca.uhn.fhir.mdm.api.IGoldenResourceMergerSvc; import ca.uhn.fhir.mdm.api.IMdmSettings; import ca.uhn.fhir.mdm.api.IMdmSurvivorshipService; import ca.uhn.fhir.mdm.log.Logs; @@ -38,33 +66,8 @@ import ca.uhn.fhir.mdm.provider.MdmProviderLoader; import ca.uhn.fhir.mdm.rules.config.MdmRuleValidator; import ca.uhn.fhir.mdm.rules.svc.MdmResourceMatcherSvc; import ca.uhn.fhir.mdm.util.EIDHelper; -import ca.uhn.fhir.mdm.util.MessageHelper; import ca.uhn.fhir.mdm.util.GoldenResourceHelper; -import ca.uhn.fhir.jpa.dao.mdm.MdmLinkDeleteSvc; -import ca.uhn.fhir.jpa.mdm.broker.MdmMessageHandler; -import ca.uhn.fhir.jpa.mdm.broker.MdmQueueConsumerLoader; -import ca.uhn.fhir.jpa.mdm.dao.MdmLinkDaoSvc; -import ca.uhn.fhir.jpa.mdm.dao.MdmLinkFactory; -import ca.uhn.fhir.jpa.mdm.interceptor.MdmStorageInterceptor; -import ca.uhn.fhir.jpa.mdm.interceptor.IMdmStorageInterceptor; -import ca.uhn.fhir.jpa.mdm.svc.MdmClearSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmControllerSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmEidUpdateService; -import ca.uhn.fhir.jpa.mdm.svc.MdmLinkQuerySvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmLinkSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmLinkUpdaterSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmMatchFinderSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmMatchLinkSvc; -import ca.uhn.fhir.jpa.mdm.svc.MdmGoldenResourceDeletingSvc; -import ca.uhn.fhir.jpa.mdm.svc.GoldenResourceMergerSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmResourceDaoSvc; -import ca.uhn.fhir.jpa.mdm.svc.MdmResourceFilteringSvc; -import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmCandidateSearchCriteriaBuilderSvc; -import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmCandidateSearchSvc; -import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmGoldenResourceFindingSvc; -import ca.uhn.fhir.jpa.mdm.svc.candidate.FindCandidateByEidSvc; -import ca.uhn.fhir.jpa.mdm.svc.candidate.FindCandidateByLinkSvc; -import ca.uhn.fhir.jpa.mdm.svc.candidate.FindCandidateByExampleSvc; +import ca.uhn.fhir.mdm.util.MessageHelper; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; import ca.uhn.fhir.validation.IResourceLoader; import org.slf4j.Logger; @@ -189,6 +192,11 @@ public class MdmConsumerConfig { return new MdmCandidateSearchSvc(); } + @Bean + CandidateSearcher candidateSearcher(DaoRegistry theDaoRegistry, IMdmSettings theMdmSettings, MdmSearchParamSvc theMdmSearchParamSvc) { + return new CandidateSearcher(theDaoRegistry, theMdmSettings, theMdmSearchParamSvc); + } + @Bean MdmCandidateSearchCriteriaBuilderSvc mdmCriteriaBuilderSvc() { return new MdmCandidateSearchCriteriaBuilderSvc(); diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/CandidateSearcher.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/CandidateSearcher.java new file mode 100644 index 00000000000..c43e93e16c1 --- /dev/null +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/CandidateSearcher.java @@ -0,0 +1,49 @@ +package ca.uhn.fhir.jpa.mdm.svc.candidate; + +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; +import ca.uhn.fhir.jpa.mdm.svc.MdmSearchParamSvc; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.mdm.api.IMdmSettings; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Optional; + +public class CandidateSearcher { + private static final Logger ourLog = LoggerFactory.getLogger(CandidateSearcher.class); + private final DaoRegistry myDaoRegistry; + private final IMdmSettings myMdmSettings; + private final MdmSearchParamSvc myMdmSearchParamSvc; + + @Autowired + public CandidateSearcher(DaoRegistry theDaoRegistry, IMdmSettings theMdmSettings, MdmSearchParamSvc theMdmSearchParamSvc) { + myDaoRegistry = theDaoRegistry; + myMdmSettings = theMdmSettings; + myMdmSearchParamSvc = theMdmSearchParamSvc; + } + + /** + * Perform a search for mdm candidates. + * + * @param theResourceType the type of resources searched on + * @param theResourceCriteria the criteria used to search for the candidates + * @return Optional.empty() if >= IMdmSettings.getCandidateSearchLimit() candidates are found, otherwise + * return the bundle provider for the search results. + */ + public Optional search(String theResourceType, String theResourceCriteria) { + SearchParameterMap searchParameterMap = myMdmSearchParamSvc.mapFromCriteria(theResourceType, theResourceCriteria); + + searchParameterMap.setLoadSynchronousUpTo(myMdmSettings.getCandidateSearchLimit()); + + IFhirResourceDao resourceDao = myDaoRegistry.getResourceDao(theResourceType); + IBundleProvider retval = resourceDao.search(searchParameterMap); + + if (retval.size() != null && retval.size() >= myMdmSettings.getCandidateSearchLimit()) { + return Optional.empty(); + } + return Optional.of(retval); + } +} diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/MdmCandidateSearchSvc.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/MdmCandidateSearchSvc.java index 4b3a0e17d4e..11ea12c1c5a 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/MdmCandidateSearchSvc.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/MdmCandidateSearchSvc.java @@ -20,11 +20,7 @@ package ca.uhn.fhir.jpa.mdm.svc.candidate; * #L% */ -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.dao.index.IdHelperService; -import ca.uhn.fhir.jpa.mdm.svc.MdmSearchParamSvc; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.mdm.api.IMdmSettings; import ca.uhn.fhir.mdm.log.Logs; import ca.uhn.fhir.mdm.rules.json.MdmFilterSearchParamJson; @@ -54,13 +50,11 @@ public class MdmCandidateSearchSvc { @Autowired private IMdmSettings myMdmSettings; @Autowired - private MdmSearchParamSvc myMdmSearchParamSvc; - @Autowired - private DaoRegistry myDaoRegistry; - @Autowired private IdHelperService myIdHelperService; @Autowired private MdmCandidateSearchCriteriaBuilderSvc myMdmCandidateSearchCriteriaBuilderSvc; + @Autowired + private CandidateSearcher myCandidateSearcher; public MdmCandidateSearchSvc() { } @@ -128,15 +122,11 @@ public class MdmCandidateSearchSvc { ourLog.debug("Searching for {} candidates with {}", theResourceType, resourceCriteria); //2. - SearchParameterMap searchParameterMap = myMdmSearchParamSvc.mapFromCriteria(theResourceType, resourceCriteria); - - searchParameterMap.setLoadSynchronous(true); - - //TODO MDM this will blow up under large scale i think. - //3. - IFhirResourceDao resourceDao = myDaoRegistry.getResourceDao(theResourceType); - IBundleProvider search = resourceDao.search(searchParameterMap); - List resources = search.getAllResources(); + Optional bundleProvider = myCandidateSearcher.search(theResourceType, resourceCriteria); + if (!bundleProvider.isPresent()) { + throw new TooManyCandidatesException("More than " + myMdmSettings.getCandidateSearchLimit() + " candidate matches found for " + resourceCriteria + ". Aborting mdm matching."); + } + List resources = bundleProvider.get().getAllResources(); int initialSize = theMatchedPidsToResources.size(); diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/TooManyCandidatesException.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/TooManyCandidatesException.java new file mode 100644 index 00000000000..a9fa01d2ff5 --- /dev/null +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/candidate/TooManyCandidatesException.java @@ -0,0 +1,7 @@ +package ca.uhn.fhir.jpa.mdm.svc.candidate; + +public class TooManyCandidatesException extends RuntimeException { + public TooManyCandidatesException(String theMessage) { + super(theMessage); + } +} diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmStorageInterceptorIT.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmStorageInterceptorIT.java index c06f0db2c7f..c722b23dab2 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmStorageInterceptorIT.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmStorageInterceptorIT.java @@ -119,6 +119,9 @@ public class MdmStorageInterceptorIT extends BaseMdmR4Test { } } + // TODO This test often fails in IntelliJ with the error message: + // "The operation has failed with a version constraint failure. This generally means that two clients/threads were + // trying to update the same resource at the same time, and this request was chosen as the failing request." @Test public void testCreatingGoldenResourceWithInsufficentMDMAttributesIsNotMDMProcessed() throws InterruptedException { myMdmHelper.doCreateResource(new Patient(), true); @@ -200,7 +203,7 @@ public class MdmStorageInterceptorIT extends BaseMdmR4Test { // Updating a Golden Resource Patient who was created via MDM should fail. MdmLink mdmLink = myMdmLinkDaoSvc.getMatchedLinkForSourcePid(myIdHelperService.getPidOrNull(patient)).get(); Long sourcePatientPid = mdmLink.getGoldenResourcePid(); - Patient goldenResourcePatient = (Patient) myPatientDao.readByPid(new ResourcePersistentId(sourcePatientPid)); + Patient goldenResourcePatient = myPatientDao.readByPid(new ResourcePersistentId(sourcePatientPid)); goldenResourcePatient.setGender(Enumerations.AdministrativeGender.MALE); try { myMdmHelper.doUpdateResource(goldenResourcePatient, true); diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmCandidateSearchSvcTest.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmCandidateSearchSvcIT.java similarity index 64% rename from hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmCandidateSearchSvcTest.java rename to hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmCandidateSearchSvcIT.java index 55288727bb1..3d9e7f2b6d3 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmCandidateSearchSvcTest.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/MdmCandidateSearchSvcIT.java @@ -2,10 +2,13 @@ package ca.uhn.fhir.jpa.mdm.svc; import ca.uhn.fhir.jpa.mdm.BaseMdmR4Test; import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmCandidateSearchSvc; +import ca.uhn.fhir.jpa.mdm.svc.candidate.TooManyCandidatesException; +import ca.uhn.fhir.mdm.rules.config.MdmSettings; import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Practitioner; import org.hl7.fhir.r4.model.Reference; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -16,17 +19,23 @@ import java.util.Date; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; -public class MdmCandidateSearchSvcTest extends BaseMdmR4Test { +public class MdmCandidateSearchSvcIT extends BaseMdmR4Test { @Autowired MdmCandidateSearchSvc myMdmCandidateSearchSvc; + @Autowired + MdmSettings myMdmSettings; + + @AfterEach + public void resetMdmSettings() { + myMdmSettings.setCandidateSearchLimit(MdmSettings.DEFAULT_CANDIDATE_SEARCH_LIMIT); + } @Test public void testFindCandidates() { - Patient jane = buildJanePatient(); - jane.setActive(true); - createPatient(jane); + createActivePatient(); Patient newJane = buildJanePatient(); Collection result = myMdmCandidateSearchSvc.findCandidates("Patient", newJane); @@ -65,4 +74,30 @@ public class MdmCandidateSearchSvcTest extends BaseMdmR4Test { Collection patient = myMdmCandidateSearchSvc.findCandidates("Patient", incomingPatient); assertThat(patient, hasSize(1)); } + + @Test + public void testTooManyMatches() { + myMdmSettings.setCandidateSearchLimit(3); + + Patient newJane = buildJanePatient(); + + createActivePatient(); + assertEquals(1, myMdmCandidateSearchSvc.findCandidates("Patient", newJane).size()); + createActivePatient(); + assertEquals(2, myMdmCandidateSearchSvc.findCandidates("Patient", newJane).size()); + + try { + createActivePatient(); + myMdmCandidateSearchSvc.findCandidates("Patient", newJane); + fail(); + } catch (TooManyCandidatesException e) { + assertEquals("More than 3 candidate matches found for Patient?identifier=http%3A%2F%2Fa.tv%2F%7CID.JANE.123&active=true. Aborting mdm matching.", e.getMessage()); + } + } + + private Patient createActivePatient() { + Patient jane = buildJanePatient(); + jane.setActive(true); + return createPatient(jane); + } } diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/candidate/CandidateSearcherTest.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/candidate/CandidateSearcherTest.java new file mode 100644 index 00000000000..18befab3b40 --- /dev/null +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/svc/candidate/CandidateSearcherTest.java @@ -0,0 +1,67 @@ +package ca.uhn.fhir.jpa.mdm.svc.candidate; + +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; +import ca.uhn.fhir.jpa.mdm.svc.MdmSearchParamSvc; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.mdm.api.IMdmRuleValidator; +import ca.uhn.fhir.mdm.rules.config.MdmSettings; +import ca.uhn.fhir.rest.api.server.IBundleProvider; +import ca.uhn.fhir.rest.server.SimpleBundleProvider; +import org.hl7.fhir.r4.model.Patient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class CandidateSearcherTest { + @Mock + DaoRegistry myDaoRegistry; + @Mock + private IMdmRuleValidator myMdmRuleValidator; + private final MdmSettings myMdmSettings = new MdmSettings(myMdmRuleValidator); + @Mock + private MdmSearchParamSvc myMdmSearchParamSvc; + private CandidateSearcher myCandidateSearcher; + + @BeforeEach + public void before() { + myCandidateSearcher = new CandidateSearcher(myDaoRegistry, myMdmSettings, myMdmSearchParamSvc); + } + + @ParameterizedTest + @ValueSource(ints = {-1, 0, +1}) + public void testSearchLimit(int offset) { + // setup + String criteria = "?active=true"; + SearchParameterMap map = new SearchParameterMap(); + String resourceType = "Patient"; + when(myMdmSearchParamSvc.mapFromCriteria(resourceType, criteria)).thenReturn(map); + IFhirResourceDao dao = mock(IFhirResourceDao.class); + when(myDaoRegistry.getResourceDao(resourceType)).thenReturn(dao); + int candidateSearchLimit = 2401; + myMdmSettings.setCandidateSearchLimit(candidateSearchLimit); + SimpleBundleProvider bundleProvider = new SimpleBundleProvider(); + + bundleProvider.setSize(candidateSearchLimit + offset); + when(dao.search(map)).thenReturn(bundleProvider); + + Optional result = myCandidateSearcher.search(resourceType, criteria); + + // validate + assertTrue(map.isLoadSynchronous()); + assertEquals(candidateSearchLimit, map.getLoadSynchronousUpTo()); + boolean shouldNotFailBecauseOfTooManyMatches = offset < 0; + assertTrue(result.isPresent() == shouldNotFailBecauseOfTooManyMatches); + } +} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java index 83bf4343d89..fd1dbd86f2f 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java @@ -52,4 +52,6 @@ public interface IMdmSettings { default String getSupportedMdmTypes() { return getMdmRules().getMdmTypes().stream().collect(Collectors.joining(", ")); } + + int getCandidateSearchLimit(); } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmSettings.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmSettings.java index 80e092fe4ff..803c8ed0dd3 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmSettings.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/rules/config/MdmSettings.java @@ -31,10 +31,11 @@ import java.io.IOException; @Component public class MdmSettings implements IMdmSettings { + public static final int DEFAULT_CANDIDATE_SEARCH_LIMIT = 10000; private final IMdmRuleValidator myMdmRuleValidator; private boolean myEnabled; - private int myConcurrentConsumers = MDM_DEFAULT_CONCURRENT_CONSUMERS; + private final int myConcurrentConsumers = MDM_DEFAULT_CONCURRENT_CONSUMERS; private String myScriptText; private String mySurvivorshipRules; private MdmRulesJson myMdmRules; @@ -42,12 +43,18 @@ public class MdmSettings implements IMdmSettings { /** * If disabled, the underlying MDM system will operate under the following assumptions: - * + *

* 1. Source resource may have more than 1 EID of the same system simultaneously. * 2. During linking, incoming patient EIDs will be merged with existing Golden Resource EIDs. */ private boolean myPreventMultipleEids; + /** + * When searching for matching candidates, this is the maximum number of candidates that will be retrieved. If the + * number matched is equal to or higher than this, then an exception will be thrown and candidate matching will be aborted + */ + private int myCandidateSearchLimit = DEFAULT_CANDIDATE_SEARCH_LIMIT; + @Autowired public MdmSettings(IMdmRuleValidator theMdmRuleValidator) { myMdmRuleValidator = theMdmRuleValidator; @@ -121,4 +128,13 @@ public class MdmSettings implements IMdmSettings { public void setSurvivorshipRules(String theSurvivorshipRules) { mySurvivorshipRules = theSurvivorshipRules; } + + @Override + public int getCandidateSearchLimit() { + return myCandidateSearchLimit; + } + + public void setCandidateSearchLimit(int theCandidateSearchLimit) { + myCandidateSearchLimit = theCandidateSearchLimit; + } }