From 78ce2a6344c433004277da3ece9ac5d1d33d10c3 Mon Sep 17 00:00:00 2001 From: Tadgh Date: Tue, 25 Oct 2022 12:02:04 -0700 Subject: [PATCH] 4182 convert $mdm submit to a batch job (#4188) * Changelog * Wip implementation * Wip * wip * wip * Fix validator * Fix up implementation * tidfy * TIdy up tests * Add some docs * Add changelog info: * Tidy up self review * License files * Update hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md Co-authored-by: Ken Stevens * Review changes * fixes * add new method * Update tests * Add batch size parameter Co-authored-by: Ken Stevens --- .../6_2_0/4182-mdm-submit-batch.yaml | 4 + .../docs/server_jpa_mdm/mdm_operations.md | 8 ++ .../jpa/search/helper/SearchParamHelper.java | 81 ---------------- .../jpa/mdm/svc/MdmControllerSvcImpl.java | 27 ++++++ .../mdm/provider/MdmProviderBatchR4Test.java | 65 +++++++++---- .../src/test/resources/logback-test.xml | 6 ++ .../util/SearchParameterHelper.java | 20 ++++ hapi-fhir-server-mdm/pom.xml | 2 +- .../uhn/fhir/mdm/api/IMdmControllerSvc.java | 1 + .../mdm/provider/MdmProviderDstu3Plus.java | 68 +++++++++++--- .../fhir/mdm/provider/MdmProviderLoader.java | 3 +- .../server/provider/ProviderConstants.java | 2 +- .../server/servlet/ServletRequestDetails.java | 11 +++ .../batch2/jobs/config/Batch2JobsConfig.java | 3 +- .../batch2/jobs/export/BulkExportAppCtx.java | 8 +- .../jobs/export/ExpandResourcesStep.java | 19 ++-- .../jobs/export/FetchResourceIdsStep.java | 10 +- .../batch2/jobs/export/WriteBinaryStep.java | 8 +- ...ources.java => ExpandedResourcesList.java} | 3 +- ...kExportIdList.java => ResourceIdList.java} | 2 +- .../jobs/export/ExpandResourcesStepTest.java | 22 ++--- .../jobs/export/FetchResourceIdsStepTest.java | 18 ++-- .../jobs/export/WriteBinaryStepTest.java | 17 ++-- .../chunk/ResourceIdListWorkChunkJson.java | 6 +- .../fhir/batch2/jobs/step/LoadIdsStep.java | 8 +- .../batch2/jobs/step/ResourceIdListStep.java | 2 +- .../uhn/fhir/mdm/batch2/MdmBatch2Config.java | 6 +- .../MdmInflateAndSubmitResourcesStep.java | 92 +++++++++++++++++++ .../mdm/batch2/submit/MdmSubmitAppCtx.java | 82 +++++++++++++++++ .../batch2/submit/MdmSubmitJobParameters.java | 27 ++++++ .../MdmSubmitJobParametersValidator.java | 92 +++++++++++++++++++ .../MdmSubmitJobParametersValidatorTest.java | 67 ++++++++++++++ 32 files changed, 611 insertions(+), 179 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4182-mdm-submit-batch.yaml delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/helper/SearchParamHelper.java rename hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/{BulkExportExpandedResources.java => ExpandedResourcesList.java} (92%) rename hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/{BulkExportIdList.java => ResourceIdList.java} (95%) create mode 100644 hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmInflateAndSubmitResourcesStep.java create mode 100644 hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitAppCtx.java create mode 100644 hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParameters.java create mode 100644 hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidator.java create mode 100644 hapi-fhir-storage-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidatorTest.java diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4182-mdm-submit-batch.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4182-mdm-submit-batch.yaml new file mode 100644 index 00000000000..80595d41eab --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_2_0/4182-mdm-submit-batch.yaml @@ -0,0 +1,4 @@ +--- +type: add +issue: 4182 +title: "`$mdm-submit` can now be run as a batch job, which will return a job ID, and can be polled for status. This can be accomplished by sending a `Prefer: respond-async` header with the request." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md index cbe152e85f5..ca100186c84 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md @@ -718,6 +718,10 @@ After the operation is complete, all resources that matched the criteria will no This operation takes a single optional criteria parameter unless it is called on a specific instance. +Note that this operation can take a long time on large data sets. In order to support large data sets, the operation can be run asynchronously. This can be done by +sending the `Prefer: respond-async` header with the request. This will cause HAPI-FHIR to execute the request as a batch job. The response will contain a `jobId` parameter that can be used to poll the status of the operation. Note that completion of the job indicates completion of loading all the resources onto the broker, +not necessarily the completion of the actual underlying MDM process. + @@ -773,9 +777,13 @@ This operation returns the number of resources that were submitted for MDM proce } ``` + + This operation can also be done at the Instance level. When this is the case, the operations accepts no parameters. The following are examples of Instance level POSTs, which require no parameters. ```url http://example.com/Patient/123/$mdm-submit http://example.com/Practitioner/456/$mdm-submit ``` + + diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/helper/SearchParamHelper.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/helper/SearchParamHelper.java deleted file mode 100644 index 417270968da..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/helper/SearchParamHelper.java +++ /dev/null @@ -1,81 +0,0 @@ -package ca.uhn.fhir.jpa.search.helper; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.context.RuntimeResourceDefinition; -import ca.uhn.fhir.context.RuntimeSearchParam; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Service -public class SearchParamHelper { - - @Autowired - private FhirContext myFhirContext; - - - public Collection getPatientSearchParamsForResourceType(String theResourceType) { - RuntimeResourceDefinition runtimeResourceDefinition = myFhirContext.getResourceDefinition(theResourceType); - Map searchParams = new HashMap<>(); - - RuntimeSearchParam patientSearchParam = runtimeResourceDefinition.getSearchParam("patient"); - if (patientSearchParam != null) { - searchParams.put(patientSearchParam.getName(), patientSearchParam); - - } - RuntimeSearchParam subjectSearchParam = runtimeResourceDefinition.getSearchParam("subject"); - if (subjectSearchParam != null) { - searchParams.put(subjectSearchParam.getName(), subjectSearchParam); - } - - List compartmentSearchParams = getPatientCompartmentRuntimeSearchParams(runtimeResourceDefinition); - compartmentSearchParams.forEach(param -> searchParams.put(param.getName(), param)); - - return searchParams.values(); - } - - /** - * Search the resource definition for a compartment named 'patient' and return its related Search Parameter. - */ - public List getPatientCompartmentRuntimeSearchParams(RuntimeResourceDefinition runtimeResourceDefinition) { - List patientSearchParam = new ArrayList<>(); - List searchParams = runtimeResourceDefinition.getSearchParamsForCompartmentName("Patient"); - return searchParams; -// if (searchParams == null || searchParams.size() == 0) { -// String errorMessage = String.format("Resource type [%s] is not eligible for this type of export, as it contains no Patient compartment, and no `patient` or `subject` search parameter", myResourceType); -// throw new IllegalArgumentException(Msg.code(1264) + errorMessage); -// } else if (searchParams.size() == 1) { -// patientSearchParam = searchParams.get(0); -// } else { -// String errorMessage = String.format("Resource type [%s] is not eligible for Group Bulk export, as we are unable to disambiguate which patient search parameter we should be searching by.", myResourceType); -// throw new IllegalArgumentException(Msg.code(1265) + errorMessage); -// } -// return patientSearchParam; - } -} diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java index dee4d91044e..caccbd450ab 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java @@ -38,6 +38,8 @@ import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; import ca.uhn.fhir.mdm.api.paging.MdmPageRequest; import ca.uhn.fhir.mdm.batch2.clear.MdmClearAppCtx; import ca.uhn.fhir.mdm.batch2.clear.MdmClearJobParameters; +import ca.uhn.fhir.mdm.batch2.submit.MdmSubmitAppCtx; +import ca.uhn.fhir.mdm.batch2.submit.MdmSubmitJobParameters; import ca.uhn.fhir.mdm.model.MdmTransactionContext; import ca.uhn.fhir.mdm.provider.MdmControllerHelper; import ca.uhn.fhir.mdm.provider.MdmControllerUtil; @@ -196,6 +198,31 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc { return retVal; } + + @Override + public IBaseParameters submitMdmSubmitJob(List theUrls, IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails) { + MdmSubmitJobParameters params = new MdmSubmitJobParameters(); + + if (theBatchSize != null && theBatchSize.getValue() !=null && theBatchSize.getValue().longValue() > 0) { + params.setBatchSize(theBatchSize.getValue().intValue()); + } + ReadPartitionIdRequestDetails details= new ReadPartitionIdRequestDetails(null, RestOperationTypeEnum.EXTENDED_OPERATION_SERVER, null, null, null); + params.setRequestPartitionId(RequestPartitionId.allPartitions()); + + theUrls.forEach(params::addUrl); + + JobInstanceStartRequest request = new JobInstanceStartRequest(); + request.setParameters(params); + request.setJobDefinitionId(MdmSubmitAppCtx.MDM_SUBMIT_JOB); + + Batch2JobStartResponse batch2JobStartResponse = myJobCoordinator.startInstance(request); + String id = batch2JobStartResponse.getJobId(); + + IBaseParameters retVal = ParametersUtil.newInstance(myFhirContext); + ParametersUtil.addParameterToParametersString(myFhirContext, retVal, ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, id); + return retVal; + } + @Override public void notDuplicateGoldenResource(String theGoldenResourceId, String theTargetGoldenResourceId, MdmTransactionContext theMdmTransactionContext) { IAnyResource goldenResource = myMdmControllerHelper.getLatestGoldenResourceFromIdOrThrowException(ProviderConstants.MDM_UPDATE_LINK_GOLDEN_RESOURCE_ID, theGoldenResourceId); diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java index 99ab943ebfa..f1af3820c99 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java @@ -3,9 +3,11 @@ package ca.uhn.fhir.jpa.mdm.provider; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.api.IInterceptorService; import ca.uhn.fhir.interceptor.api.Pointcut; +import ca.uhn.fhir.jpa.partition.SystemRequestDetails; import ca.uhn.fhir.mdm.rules.config.MdmSettings; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.test.concurrency.PointcutLatch; import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.r4.model.IdType; @@ -15,16 +17,28 @@ import org.hl7.fhir.r4.model.Practitioner; import org.hl7.fhir.r4.model.StringType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; +import javax.servlet.Servlet; import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MdmProviderBatchR4Test extends BaseLinkR4Test { public static final String ORGANIZATION_DUMMY = "Organization/dummy"; @@ -44,6 +58,16 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { PointcutLatch afterMdmLatch = new PointcutLatch(Pointcut.MDM_AFTER_PERSISTED_RESOURCE_CHECKED); + public static Stream requestTypes() { + ServletRequestDetails asyncSrd = mock(ServletRequestDetails.class); + when(asyncSrd.getHeader("Prefer")).thenReturn("respond-async"); + ServletRequestDetails syncSrd = mock(ServletRequestDetails.class); + + return Stream.of( + Arguments.of(Named.of("Asynchronous Request", asyncSrd)), + Arguments.of(Named.of("Synchronous Request", syncSrd)) + ); + } @Override @BeforeEach public void before() throws Exception { @@ -74,21 +98,23 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { super.after(); } - @Test - public void testBatchRunOnAllMedications() throws InterruptedException { + @ParameterizedTest + @MethodSource("requestTypes") + public void testBatchRunOnAllMedications(ServletRequestDetails theSyncOrAsyncRequest) throws InterruptedException { StringType criteria = null; clearMdmLinks(); - afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchOnAllSourceResources(new StringType("Medication"), criteria, null)); + afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchOnAllSourceResources(new StringType("Medication"), criteria, null, theSyncOrAsyncRequest)); assertLinkCount(1); } - @Test - public void testBatchRunOnAllPractitioners() throws InterruptedException { + @ParameterizedTest + @MethodSource("requestTypes") + public void testBatchRunOnAllPractitioners(ServletRequestDetails theSyncOrAsyncRequest) throws InterruptedException { StringType criteria = null; clearMdmLinks(); - afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPractitionerType(criteria, null)); + afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPractitionerType(criteria, null, theSyncOrAsyncRequest)); assertLinkCount(1); } @Test @@ -108,12 +134,13 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { } } - @Test - public void testBatchRunOnAllPatients() throws InterruptedException { + @ParameterizedTest + @MethodSource("requestTypes") + public void testBatchRunOnAllPatients(ServletRequestDetails theSyncOrAsyncRequest) throws InterruptedException { assertLinkCount(3); StringType criteria = null; clearMdmLinks(); - afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPatientType(criteria, null)); + afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPatientType(criteria, null, theSyncOrAsyncRequest)); assertLinkCount(1); } @@ -136,29 +163,33 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { } } - @Tag("intermittent") -// @Test - public void testBatchRunOnAllTypes() throws InterruptedException { + @ParameterizedTest + @MethodSource("requestTypes") + public void testBatchRunOnAllTypes(ServletRequestDetails theSyncOrAsyncRequest) throws InterruptedException { assertLinkCount(3); StringType criteria = new StringType(""); clearMdmLinks(); afterMdmLatch.runWithExpectedCount(3, () -> { - myMdmProvider.mdmBatchOnAllSourceResources(null, criteria, null); + myMdmProvider.mdmBatchOnAllSourceResources(null, criteria, null, theSyncOrAsyncRequest); }); assertLinkCount(3); } - @Test - public void testBatchRunOnAllTypesWithInvalidCriteria() { + @ParameterizedTest + @MethodSource("requestTypes") + public void testBatchRunOnAllTypesWithInvalidCriteria(ServletRequestDetails theSyncOrAsyncRequest) { assertLinkCount(3); StringType criteria = new StringType("death-date=2020-06-01"); clearMdmLinks(); try { - myMdmProvider.mdmBatchPractitionerType(criteria, null); + myMdmProvider.mdmBatchOnAllSourceResources(null, criteria , null, theSyncOrAsyncRequest); fail(); } catch (InvalidRequestException e) { - assertThat(e.getMessage(), is(equalTo(Msg.code(488) + "Failed to parse match URL[death-date=2020-06-01] - Resource type Practitioner does not have a parameter with name: death-date"))); + + assertThat(e.getMessage(), either( + containsString(Msg.code(2039) + "Failed to validate parameters for job"))//Async case + .or(containsString(Msg.code(488) + "Failed to parse match URL")));// Sync case } } } diff --git a/hapi-fhir-jpaserver-mdm/src/test/resources/logback-test.xml b/hapi-fhir-jpaserver-mdm/src/test/resources/logback-test.xml index aa595257758..15520bd42e9 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/resources/logback-test.xml +++ b/hapi-fhir-jpaserver-mdm/src/test/resources/logback-test.xml @@ -4,6 +4,7 @@ TRACE + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%file:%line] %msg%n @@ -71,6 +72,11 @@ + + + + + diff --git a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java index abc6ae5a723..dac2781701e 100644 --- a/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java +++ b/hapi-fhir-jpaserver-searchparam/src/main/java/ca/uhn/fhir/jpa/searchparam/util/SearchParameterHelper.java @@ -1,5 +1,25 @@ package ca.uhn.fhir.jpa.searchparam.util; +/*- + * #%L + * HAPI FHIR Search Parameters + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + import ca.uhn.fhir.context.RuntimeSearchParam; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.registry.SearchParameterCanonicalizer; diff --git a/hapi-fhir-server-mdm/pom.xml b/hapi-fhir-server-mdm/pom.xml index 7de91f3c884..81b2b950f63 100644 --- a/hapi-fhir-server-mdm/pom.xml +++ b/hapi-fhir-server-mdm/pom.xml @@ -116,7 +116,7 @@ test - + diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java index 4cd57706504..e714b9a9f6b 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java @@ -55,4 +55,5 @@ public interface IMdmControllerSvc { IAnyResource createLink(String theGoldenResourceId, String theSourceResourceId, @Nullable String theMatchResult, MdmTransactionContext theMdmTransactionContext); IBaseParameters submitMdmClearJob(List theResourceNames, IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails); + IBaseParameters submitMdmSubmitJob(List theUrls, IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails); } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java index af777390d6c..82f527a8c27 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java @@ -49,8 +49,10 @@ import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.slf4j.Logger; import org.springframework.data.domain.Page; +import javax.annotation.Nonnull; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -78,7 +80,8 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { IMdmControllerSvc theMdmControllerSvc, MdmControllerHelper theMdmHelper, IMdmSubmitSvc theMdmSubmitSvc, - IMdmSettings theIMdmSettings) { + IMdmSettings theIMdmSettings + ) { super(theFhirContext); myMdmControllerSvc = theMdmControllerSvc; myMdmControllerHelper = theMdmHelper; @@ -148,7 +151,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { @OperationParam(name = ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, typeName = "decimal") }) public IBaseParameters clearMdmLinks(@OperationParam(name = ProviderConstants.OPERATION_MDM_CLEAR_RESOURCE_NAME, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theResourceNames, - @OperationParam(name = ProviderConstants.OPERATION_MDM_CLEAR_BATCH_SIZE, typeName = "decimal", min = 0, max = 1) IPrimitiveType theBatchSize, + @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_SIZE, typeName = "decimal", min = 0, max = 1) IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails) { List resourceNames = new ArrayList<>(); @@ -234,20 +237,43 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { public IBaseParameters mdmBatchOnAllSourceResources( @OperationParam(name = ProviderConstants.MDM_BATCH_RUN_RESOURCE_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType theResourceType, @OperationParam(name = ProviderConstants.MDM_BATCH_RUN_CRITERIA, min = 0, max = 1, typeName = "string") IPrimitiveType theCriteria, + @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_SIZE, typeName = "decimal", min = 0, max = 1) IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails) { String criteria = convertStringTypeToString(theCriteria); String resourceType = convertStringTypeToString(theResourceType); long submittedCount; - if (resourceType != null) { - submittedCount = myMdmSubmitSvc.submitSourceResourceTypeToMdm(resourceType, criteria, theRequestDetails); + if (theRequestDetails.isPreferRespondAsync()) { + List urls = buildUrlsForJob(criteria, resourceType); + return myMdmControllerSvc.submitMdmSubmitJob(urls, theBatchSize, theRequestDetails); } else { - submittedCount = myMdmSubmitSvc.submitAllSourceTypesToMdm(criteria, theRequestDetails); + if (StringUtils.isNotBlank(resourceType)) { + submittedCount = myMdmSubmitSvc.submitSourceResourceTypeToMdm(resourceType, criteria, theRequestDetails); + } else { + submittedCount = myMdmSubmitSvc.submitAllSourceTypesToMdm(criteria, theRequestDetails); + } + return buildMdmOutParametersWithCount(submittedCount); } - return buildMdmOutParametersWithCount(submittedCount); + } + @Nonnull + private List buildUrlsForJob(String criteria, String resourceType) { + List urls = new ArrayList<>(); + if (StringUtils.isNotBlank(resourceType)) { + String theUrl = resourceType + "?" + criteria; + urls.add(theUrl); + } else { + myMdmSettings.getMdmRules().getMdmTypes() + .stream() + .map(type -> type + "?" + criteria) + .forEach(urls::add); + } + return urls; + } + + private String convertStringTypeToString(IPrimitiveType theCriteria) { - return theCriteria == null ? null : theCriteria.getValueAsString(); + return theCriteria == null ? "" : theCriteria.getValueAsString(); } @@ -266,10 +292,16 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { }) public IBaseParameters mdmBatchPatientType( @OperationParam(name = ProviderConstants.MDM_BATCH_RUN_CRITERIA, typeName = "string") IPrimitiveType theCriteria, - RequestDetails theRequest) { - String criteria = convertStringTypeToString(theCriteria); - long submittedCount = myMdmSubmitSvc.submitPatientTypeToMdm(criteria, theRequest); - return buildMdmOutParametersWithCount(submittedCount); + @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_SIZE, typeName = "decimal", min = 0, max = 1) IPrimitiveType theBatchSize, + ServletRequestDetails theRequest) { + if (theRequest.isPreferRespondAsync()) { + String theUrl = "Patient?"; + return myMdmControllerSvc.submitMdmSubmitJob(Collections.singletonList(theUrl), theBatchSize, theRequest); + } else { + String criteria = convertStringTypeToString(theCriteria); + long submittedCount = myMdmSubmitSvc.submitPatientTypeToMdm(criteria, theRequest); + return buildMdmOutParametersWithCount(submittedCount); + } } @Operation(name = ProviderConstants.OPERATION_MDM_SUBMIT, idempotent = false, typeName = "Practitioner", returnParameters = { @@ -287,10 +319,16 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { }) public IBaseParameters mdmBatchPractitionerType( @OperationParam(name = ProviderConstants.MDM_BATCH_RUN_CRITERIA, typeName = "string") IPrimitiveType theCriteria, - RequestDetails theRequest) { - String criteria = convertStringTypeToString(theCriteria); - long submittedCount = myMdmSubmitSvc.submitPractitionerTypeToMdm(criteria, theRequest); - return buildMdmOutParametersWithCount(submittedCount); + @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_SIZE, typeName = "decimal", min = 0, max = 1) IPrimitiveType theBatchSize, + ServletRequestDetails theRequest) { + if (theRequest.isPreferRespondAsync()) { + String theUrl = "Practitioner?"; + return myMdmControllerSvc.submitMdmSubmitJob(Collections.singletonList(theUrl), theBatchSize, theRequest); + } else { + String criteria = convertStringTypeToString(theCriteria); + long submittedCount = myMdmSubmitSvc.submitPractitionerTypeToMdm(criteria, theRequest); + return buildMdmOutParametersWithCount(submittedCount); + } } /** diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java index e37ebf36e09..501c80cd14a 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java @@ -57,7 +57,8 @@ public class MdmProviderLoader { myMdmControllerSvc, myMdmControllerHelper, myMdmSubmitSvc, - myMdmSettings)); + myMdmSettings + )); break; default: throw new ConfigurationException(Msg.code(1497) + "MDM not supported for FHIR version " + myFhirContext.getVersion().getVersion()); diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java index efecaec3068..cb533ece5de 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java @@ -97,7 +97,7 @@ public class ProviderConstants { public static final String OPERATION_MDM_CLEAR = "$mdm-clear"; public static final String OPERATION_MDM_CLEAR_RESOURCE_NAME = "resourceType"; - public static final String OPERATION_MDM_CLEAR_BATCH_SIZE = "batchSize"; + public static final String OPERATION_MDM_BATCH_SIZE = "batchSize"; public static final String OPERATION_MDM_SUBMIT = "$mdm-submit"; public static final String OPERATION_MDM_SUBMIT_OUT_PARAM_SUBMITTED = "submitted"; public static final String MDM_BATCH_RUN_CRITERIA = "criteria"; diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/servlet/ServletRequestDetails.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/servlet/ServletRequestDetails.java index 8115a6791d2..9cf18c6bfa1 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/servlet/ServletRequestDetails.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/servlet/ServletRequestDetails.java @@ -24,8 +24,10 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.PreferHeader; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.server.RestfulServer; +import ca.uhn.fhir.rest.server.RestfulServerUtils; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.Validate; @@ -206,4 +208,13 @@ public class ServletRequestDetails extends RequestDetails { return Collections.unmodifiableMap(retVal); } + /** + * Returns true if the `Prefer` header contains a value of `respond-async` + */ + public boolean isPreferRespondAsync() { + String preferHeader = getHeader(Constants.HEADER_PREFER); + PreferHeader prefer = RestfulServerUtils.parsePreferHeader(null, preferHeader); + return prefer.getRespondAsync(); + } + } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java index 1f06a0603b1..c8ce12c2980 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java @@ -38,7 +38,8 @@ import org.springframework.context.annotation.Import; DeleteExpungeAppCtx.class, BulkExportAppCtx.class, TermCodeSystemJobConfig.class, - BulkImportPullConfig.class + BulkImportPullConfig.class, }) public class Batch2JobsConfig { + } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportAppCtx.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportAppCtx.java index f892bb28150..cdcc06a5f7b 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportAppCtx.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportAppCtx.java @@ -22,8 +22,8 @@ package ca.uhn.fhir.batch2.jobs.export; import ca.uhn.fhir.batch2.api.VoidModel; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportBinaryFileId; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportExpandedResources; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportIdList; +import ca.uhn.fhir.batch2.jobs.export.models.ExpandedResourcesList; +import ca.uhn.fhir.batch2.jobs.export.models.ResourceIdList; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.jpa.api.model.BulkExportJobResults; @@ -51,14 +51,14 @@ public class BulkExportAppCtx { .addFirstStep( "fetch-resources", "Fetches resource PIDs for exporting", - BulkExportIdList.class, + ResourceIdList.class, fetchResourceIdsStep() ) // expand out - fetch resources .addIntermediateStep( "expand-resources", "Expand out resources", - BulkExportExpandedResources.class, + ExpandedResourcesList.class, expandResourcesStep() ) // write binaries and save to db diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java index 522858e1159..4ae86254af5 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java @@ -25,8 +25,8 @@ import ca.uhn.fhir.batch2.api.IJobStepWorker; import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportExpandedResources; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportIdList; +import ca.uhn.fhir.batch2.jobs.export.models.ExpandedResourcesList; +import ca.uhn.fhir.batch2.jobs.export.models.ResourceIdList; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.batch2.jobs.models.Id; import ca.uhn.fhir.context.FhirContext; @@ -38,20 +38,17 @@ import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.server.interceptor.ResponseTerminologyTranslationSvc; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimap; import org.hl7.fhir.instance.model.api.IBaseResource; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import javax.annotation.Nonnull; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static org.slf4j.LoggerFactory.getLogger; -public class ExpandResourcesStep implements IJobStepWorker { +public class ExpandResourcesStep implements IJobStepWorker { private static final Logger ourLog = getLogger(ExpandResourcesStep.class); @Autowired @@ -68,9 +65,9 @@ public class ExpandResourcesStep implements IJobStepWorker theStepExecutionDetails, - @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { - BulkExportIdList idList = theStepExecutionDetails.getData(); + public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, + @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { + ResourceIdList idList = theStepExecutionDetails.getData(); BulkExportJobParameters jobParameters = theStepExecutionDetails.getParameters(); ourLog.info("Step 2 for bulk export - Expand resources"); @@ -95,7 +92,7 @@ public class ExpandResourcesStep implements IJobStepWorker fetchAllResources(BulkExportIdList theIds) { + private List fetchAllResources(ResourceIdList theIds) { List resources = new ArrayList<>(); for (Id id : theIds.getIds()) { diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStep.java index 698b154ed81..1560638dfa0 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStep.java @@ -26,7 +26,7 @@ import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.api.VoidModel; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportIdList; +import ca.uhn.fhir.batch2.jobs.export.models.ResourceIdList; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.batch2.jobs.models.Id; import ca.uhn.fhir.i18n.Msg; @@ -45,7 +45,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -public class FetchResourceIdsStep implements IFirstJobStepWorker { +public class FetchResourceIdsStep implements IFirstJobStepWorker { private static final Logger ourLog = LoggerFactory.getLogger(FetchResourceIdsStep.class); public static final int MAX_IDS_TO_BATCH = 900; @@ -59,7 +59,7 @@ public class FetchResourceIdsStep implements IFirstJobStepWorker theStepExecutionDetails, - @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { + @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { BulkExportJobParameters params = theStepExecutionDetails.getParameters(); ourLog.info("Starting BatchExport job"); @@ -131,8 +131,8 @@ public class FetchResourceIdsStep implements IFirstJobStepWorker theIds, String theResourceType, BulkExportJobParameters theParams, - IJobDataSink theDataSink) { - BulkExportIdList idList = new BulkExportIdList(); + IJobDataSink theDataSink) { + ResourceIdList idList = new ResourceIdList(); idList.setIds(theIds); diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java index eed3681c5c0..e77f3218da0 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java @@ -26,7 +26,7 @@ import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportBinaryFileId; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportExpandedResources; +import ca.uhn.fhir.batch2.jobs.export.models.ExpandedResourcesList; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.i18n.Msg; @@ -49,7 +49,7 @@ import java.io.OutputStreamWriter; import static org.slf4j.LoggerFactory.getLogger; -public class WriteBinaryStep implements IJobStepWorker { +public class WriteBinaryStep implements IJobStepWorker { private static final Logger ourLog = getLogger(WriteBinaryStep.class); @Autowired @@ -60,10 +60,10 @@ public class WriteBinaryStep implements IJobStepWorker theStepExecutionDetails, + public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { - BulkExportExpandedResources expandedResources = theStepExecutionDetails.getData(); + ExpandedResourcesList expandedResources = theStepExecutionDetails.getData(); final int numResourcesProcessed = expandedResources.getStringifiedResources().size(); ourLog.info("Write binary step of Job Export"); diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportExpandedResources.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/ExpandedResourcesList.java similarity index 92% rename from hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportExpandedResources.java rename to hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/ExpandedResourcesList.java index 813affb0285..0a227a0cd5c 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportExpandedResources.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/ExpandedResourcesList.java @@ -21,11 +21,10 @@ package ca.uhn.fhir.batch2.jobs.export.models; */ import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Multimap; import java.util.List; -public class BulkExportExpandedResources extends BulkExportJobBase { +public class ExpandedResourcesList extends BulkExportJobBase { /** * List of stringified resources ready for writing diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportIdList.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/ResourceIdList.java similarity index 95% rename from hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportIdList.java rename to hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/ResourceIdList.java index 138bfdf5857..7c497ee6c03 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportIdList.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/ResourceIdList.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; -public class BulkExportIdList extends BulkExportJobBase { +public class ResourceIdList extends BulkExportJobBase { /** * List of Id objects for serialization diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java index 493abb66cce..745dc723c49 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStepTest.java @@ -4,8 +4,8 @@ package ca.uhn.fhir.batch2.jobs.export; import ca.uhn.fhir.batch2.api.IJobDataSink; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportExpandedResources; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportIdList; +import ca.uhn.fhir.batch2.jobs.export.models.ExpandedResourcesList; +import ca.uhn.fhir.batch2.jobs.export.models.ResourceIdList; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.batch2.jobs.models.Id; import ca.uhn.fhir.batch2.model.JobInstance; @@ -66,10 +66,10 @@ public class ExpandResourcesStepTest { return parameters; } - private StepExecutionDetails createInput(BulkExportIdList theData, - BulkExportJobParameters theParameters, - JobInstance theInstance) { - StepExecutionDetails input = new StepExecutionDetails<>( + private StepExecutionDetails createInput(ResourceIdList theData, + BulkExportJobParameters theParameters, + JobInstance theInstance) { + StepExecutionDetails input = new StepExecutionDetails<>( theParameters, theData, theInstance, @@ -90,9 +90,9 @@ public class ExpandResourcesStepTest { //setup JobInstance instance = new JobInstance(); instance.setInstanceId("1"); - IJobDataSink sink = mock(IJobDataSink.class); + IJobDataSink sink = mock(IJobDataSink.class); IFhirResourceDao patientDao = mockOutDaoRegistry(); - BulkExportIdList idList = new BulkExportIdList(); + ResourceIdList idList = new ResourceIdList(); idList.setResourceType("Patient"); ArrayList resources = new ArrayList<>(); ArrayList ids = new ArrayList<>(); @@ -109,7 +109,7 @@ public class ExpandResourcesStepTest { } idList.setIds(ids); - StepExecutionDetails input = createInput( + StepExecutionDetails input = createInput( idList, createParameters(), instance @@ -125,10 +125,10 @@ public class ExpandResourcesStepTest { // data sink - ArgumentCaptor expandedCaptor = ArgumentCaptor.forClass(BulkExportExpandedResources.class); + ArgumentCaptor expandedCaptor = ArgumentCaptor.forClass(ExpandedResourcesList.class); verify(sink) .accept(expandedCaptor.capture()); - BulkExportExpandedResources expandedResources = expandedCaptor.getValue(); + ExpandedResourcesList expandedResources = expandedCaptor.getValue(); assertEquals(resources.size(), expandedResources.getStringifiedResources().size()); // we'll only verify a single element // but we want to make sure it's as compact as possible diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStepTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStepTest.java index 0947d8b219a..a085d65adc6 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStepTest.java +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/FetchResourceIdsStepTest.java @@ -4,7 +4,7 @@ import ca.uhn.fhir.batch2.api.IJobDataSink; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.api.VoidModel; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportIdList; +import ca.uhn.fhir.batch2.jobs.export.models.ResourceIdList; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.batch2.jobs.models.Id; import ca.uhn.fhir.batch2.model.JobInstance; @@ -91,7 +91,7 @@ public class FetchResourceIdsStepTest { @Test public void run_withValidInputs_succeeds() { // setup - IJobDataSink sink = mock(IJobDataSink.class); + IJobDataSink sink = mock(IJobDataSink.class); BulkExportJobParameters parameters = createParameters(); JobInstance instance = new JobInstance(); instance.setInstanceId("1"); @@ -126,14 +126,14 @@ public class FetchResourceIdsStepTest { // verify assertEquals(RunOutcome.SUCCESS, outcome); - ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(BulkExportIdList.class); + ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(ResourceIdList.class); verify(sink, times(parameters.getResourceTypes().size())) .accept(resultCaptor.capture()); - List results = resultCaptor.getAllValues(); + List results = resultCaptor.getAllValues(); assertEquals(parameters.getResourceTypes().size(), results.size()); for (int i = 0; i < results.size(); i++) { - BulkExportIdList idList = results.get(i); + ResourceIdList idList = results.get(i); String resourceType = idList.getResourceType(); assertTrue(parameters.getResourceTypes().contains(resourceType)); @@ -165,7 +165,7 @@ public class FetchResourceIdsStepTest { @Test public void run_moreThanTheMaxFileCapacityPatients_hasAtLeastTwoJobs() { // setup - IJobDataSink sink = mock(IJobDataSink.class); + IJobDataSink sink = mock(IJobDataSink.class); JobInstance instance = new JobInstance(); instance.setInstanceId("1"); BulkExportJobParameters parameters = createParameters(); @@ -192,18 +192,18 @@ public class FetchResourceIdsStepTest { RunOutcome outcome = myFirstStep.run(input, sink); // verify - ArgumentCaptor captor = ArgumentCaptor.forClass(BulkExportIdList.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(ResourceIdList.class); assertEquals(RunOutcome.SUCCESS, outcome); verify(sink, times(2)) .accept(captor.capture()); - List listIds = captor.getAllValues(); + List listIds = captor.getAllValues(); // verify all submitted ids are there boolean found = false; for (ResourcePersistentId pid : patientIds) { Id id = Id.getIdFromPID(pid, "Patient"); - for (BulkExportIdList idList : listIds) { + for (ResourceIdList idList : listIds) { found = idList.getIds().contains(id); if (found) { break; diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java index 5e97dd52330..9b0aa800c80 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStepTest.java @@ -6,7 +6,7 @@ import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportBinaryFileId; -import ca.uhn.fhir.batch2.jobs.export.models.BulkExportExpandedResources; +import ca.uhn.fhir.batch2.jobs.export.models.ExpandedResourcesList; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.context.FhirContext; @@ -18,7 +18,6 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; -import com.google.common.collect.Multimap; import org.hl7.fhir.instance.model.api.IBaseBinary; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.IdType; @@ -97,12 +96,12 @@ public class WriteBinaryStepTest { ourLog.detachAppender(myAppender); } - private StepExecutionDetails createInput(BulkExportExpandedResources theData, - JobInstance theInstance) { + private StepExecutionDetails createInput(ExpandedResourcesList theData, + JobInstance theInstance) { BulkExportJobParameters parameters = new BulkExportJobParameters(); parameters.setStartDate(new Date()); parameters.setResourceTypes(Arrays.asList("Patient", "Observation")); - StepExecutionDetails input = new StepExecutionDetails<>( + StepExecutionDetails input = new StepExecutionDetails<>( parameters, theData, theInstance, @@ -114,7 +113,7 @@ public class WriteBinaryStepTest { @Test public void run_validInputNoErrors_succeeds() { // setup - BulkExportExpandedResources expandedResources = new BulkExportExpandedResources(); + ExpandedResourcesList expandedResources = new ExpandedResourcesList(); JobInstance instance = new JobInstance(); instance.setInstanceId("1"); List stringified = Arrays.asList("first", "second", "third", "forth"); @@ -122,7 +121,7 @@ public class WriteBinaryStepTest { expandedResources.setResourceType("Patient"); IFhirResourceDao binaryDao = mock(IFhirResourceDao.class); IJobDataSink sink = mock(IJobDataSink.class); - StepExecutionDetails input = createInput(expandedResources, instance); + StepExecutionDetails input = createInput(expandedResources, instance); IIdType binaryId = new IdType("Binary/123"); DaoMethodOutcome methodOutcome = new DaoMethodOutcome(); methodOutcome.setId(binaryId); @@ -163,13 +162,13 @@ public class WriteBinaryStepTest { String testException = "I am an exceptional exception."; JobInstance instance = new JobInstance(); instance.setInstanceId("1"); - BulkExportExpandedResources expandedResources = new BulkExportExpandedResources(); + ExpandedResourcesList expandedResources = new ExpandedResourcesList(); List stringified = Arrays.asList("first", "second", "third", "forth"); expandedResources.setStringifiedResources(stringified); expandedResources.setResourceType("Patient"); IFhirResourceDao binaryDao = mock(IFhirResourceDao.class); IJobDataSink sink = mock(IJobDataSink.class); - StepExecutionDetails input = createInput(expandedResources, instance); + StepExecutionDetails input = createInput(expandedResources, instance); ourLog.setLevel(Level.ERROR); diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/ResourceIdListWorkChunkJson.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/ResourceIdListWorkChunkJson.java index d0fd0ed26a2..005e97f78c6 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/ResourceIdListWorkChunkJson.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/ResourceIdListWorkChunkJson.java @@ -64,7 +64,11 @@ public class ResourceIdListWorkChunkJson implements IModelJson { return myTypedPids .stream() - .map(t -> new ResourcePersistentId(t.getPid())) + .map(t -> { + ResourcePersistentId retval = new ResourcePersistentId(t.getPid()); + retval.setResourceType(t.getResourceType()); + return retval; + }) .collect(Collectors.toList()); } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStep.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStep.java index cd2594710bb..faf834b9fe0 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStep.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStep.java @@ -29,17 +29,19 @@ import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrlListJobParameters; import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; +import org.slf4j.Logger; import javax.annotation.Nonnull; +import static org.slf4j.LoggerFactory.getLogger; + public class LoadIdsStep implements IJobStepWorker { - private final IBatch2DaoSvc myBatch2DaoSvc; + private static final Logger ourLog = getLogger(LoadIdsStep.class); + private final ResourceIdListStep myResourceIdListStep; public LoadIdsStep(IBatch2DaoSvc theBatch2DaoSvc) { - myBatch2DaoSvc = theBatch2DaoSvc; - IIdChunkProducer idChunkProducer = new PartitionedUrlListIdChunkProducer(theBatch2DaoSvc); myResourceIdListStep = new ResourceIdListStep<>(idChunkProducer); } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/ResourceIdListStep.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/ResourceIdListStep.java index dcacdc02acd..0b4963c9f96 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/ResourceIdListStep.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/ResourceIdListStep.java @@ -129,8 +129,8 @@ public class ResourceIdListStep { + private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); + + @Autowired + private DaoRegistry myDaoRegistry; + + @Autowired(required = false) + private ResponseTerminologyTranslationSvc myResponseTerminologyTranslationSvc; + @Autowired + private IMdmChannelSubmitterSvc myMdmChannelSubmitterSvc; + + @Nonnull + @Override + public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, + @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { + ResourceIdListWorkChunkJson idList = theStepExecutionDetails.getData(); + + ourLog.info("Final Step for $mdm-submit - Expand and submit resources"); + ourLog.info("About to expand {} resource IDs into their full resource bodies.", idList.getResourcePersistentIds().size()); + + //Inflate the resources by PID + List allResources = fetchAllResources(idList.getResourcePersistentIds()); + + //Replace the terminology + if (myResponseTerminologyTranslationSvc != null) { + myResponseTerminologyTranslationSvc.processResourcesForTerminologyTranslation(allResources); + } + + //Submit + for (IBaseResource nextResource : allResources) { + myMdmChannelSubmitterSvc.submitResourceToMdmChannel(nextResource); + } + + ourLog.info("Expanding of {} resources of type completed", idList.size()); + return new RunOutcome(allResources.size()); + } + + private List fetchAllResources(List theIds) { + List resources = new ArrayList<>(); + for (ResourcePersistentId id : theIds) { + assert id.getResourceType() != null; + IFhirResourceDao dao = myDaoRegistry.getResourceDao(id.getResourceType()); + // This should be a query, but we have PIDs, and we don't have a _pid search param. TODO GGG, figure out how to make this search by pid. + try { + resources.add(dao.readByPid(id)); + } catch (ResourceNotFoundException e) { + ourLog.warn("While attempging to send [{}] to the MDM queue, the resource was not found.", id); + } + } + return resources; + } +} diff --git a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitAppCtx.java b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitAppCtx.java new file mode 100644 index 00000000000..16f527bcd05 --- /dev/null +++ b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitAppCtx.java @@ -0,0 +1,82 @@ +package ca.uhn.fhir.mdm.batch2.submit; + +/*- + * #%L + * hapi-fhir-storage-mdm + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; +import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; +import ca.uhn.fhir.batch2.jobs.step.GenerateRangeChunksStep; +import ca.uhn.fhir.batch2.jobs.step.LoadIdsStep; +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; +import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import ca.uhn.fhir.mdm.api.IMdmSettings; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MdmSubmitAppCtx { + + private static final String MDM_SUBMIT_JOB_BEAN_NAME = "mdmSubmitJobDefinition"; + public static String MDM_SUBMIT_JOB= "MDM_SUBMIT"; + + @Bean + public GenerateRangeChunksStep submitGenerateRangeChunksStep() { + return new GenerateRangeChunksStep(); + } + + + @Bean(name = MDM_SUBMIT_JOB_BEAN_NAME) + public JobDefinition mdmSubmitJobDefinition(IBatch2DaoSvc theBatch2DaoSvc, MatchUrlService theMatchUrlService, FhirContext theFhirContext, IMdmSettings theMdmSettings) { + return JobDefinition.newBuilder() + .setJobDefinitionId(MDM_SUBMIT_JOB) + .setJobDescription("MDM Batch Submission") + .setJobDefinitionVersion(1) + .setParametersType(MdmSubmitJobParameters.class) + .setParametersValidator(mdmSubmitJobParametersValidator(theMatchUrlService, theFhirContext, theMdmSettings)) + .addFirstStep( + "generate-ranges", + "generate data ranges to submit to mdm", + PartitionedUrlChunkRangeJson.class, + submitGenerateRangeChunksStep()) + .addIntermediateStep( + "load-ids", + "Load the IDs", + ResourceIdListWorkChunkJson.class, + new LoadIdsStep(theBatch2DaoSvc)) + .addLastStep( + "inflate-and-submit-resources", + "Inflate and Submit resources", + mdmInflateAndSubmitResourcesStep()) + .build(); + } + + @Bean + public MdmSubmitJobParametersValidator mdmSubmitJobParametersValidator(MatchUrlService theMatchUrlService, FhirContext theFhirContext, IMdmSettings theMdmSettings) { + return new MdmSubmitJobParametersValidator(theMdmSettings, theMatchUrlService, theFhirContext); + } + + @Bean + public MdmInflateAndSubmitResourcesStep mdmInflateAndSubmitResourcesStep() { + return new MdmInflateAndSubmitResourcesStep(); + } + +} diff --git a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParameters.java b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParameters.java new file mode 100644 index 00000000000..9d93c14d6dc --- /dev/null +++ b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParameters.java @@ -0,0 +1,27 @@ +package ca.uhn.fhir.mdm.batch2.submit; + +/*- + * #%L + * hapi-fhir-storage-mdm + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrlListJobParameters; + +public class MdmSubmitJobParameters extends PartitionedUrlListJobParameters { + +} diff --git a/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidator.java b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidator.java new file mode 100644 index 00000000000..ca70857cc66 --- /dev/null +++ b/hapi-fhir-storage-mdm/src/main/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidator.java @@ -0,0 +1,92 @@ +package ca.uhn.fhir.mdm.batch2.submit; + +/*- + * #%L + * hapi-fhir-storage-mdm + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.batch2.api.IJobParametersValidator; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrl; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.RuntimeResourceDefinition; +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import ca.uhn.fhir.mdm.api.IMdmSettings; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; + + +public class MdmSubmitJobParametersValidator implements IJobParametersValidator { + + private IMdmSettings myMdmSettings; + private MatchUrlService myMatchUrlService; + private FhirContext myFhirContext; + + public MdmSubmitJobParametersValidator(IMdmSettings theMdmSettings, MatchUrlService theMatchUrlService, FhirContext theFhirContext) { + myMdmSettings = theMdmSettings; + myMatchUrlService = theMatchUrlService; + myFhirContext = theFhirContext; + } + + @Nonnull + @Override + public List validate(@Nonnull MdmSubmitJobParameters theParameters) { + List errorMsgs = new ArrayList<>(); + for (PartitionedUrl partitionedUrl : theParameters.getPartitionedUrls()) { + String url = partitionedUrl.getUrl(); + String resourceType = getResourceTypeFromUrl(url); + RuntimeResourceDefinition resourceDefinition = myFhirContext.getResourceDefinition(resourceType); + validateTypeIsUsedByMdm(errorMsgs, resourceType); + validateAllSearchParametersApplyToResourceType(errorMsgs, partitionedUrl, resourceType, resourceDefinition); + } + return errorMsgs; + } + + private void validateAllSearchParametersApplyToResourceType(List errorMsgs, PartitionedUrl partitionedUrl, String resourceType, RuntimeResourceDefinition resourceDefinition) { + try { + myMatchUrlService.translateMatchUrl(partitionedUrl.getUrl(), resourceDefinition); + } catch (MatchUrlService.UnrecognizedSearchParameterException e) { + String errorMsg = String.format("Search parameter %s is not recognized for resource type %s. Source error is %s", e.getParamName(), resourceType, e.getMessage()); + errorMsgs.add(errorMsg); + } catch (InvalidRequestException e) { + errorMsgs.add("Invalid request detected: " + e.getMessage()); + } + } + + private void validateTypeIsUsedByMdm(List errorMsgs, String resourceType) { + if (!myMdmSettings.isSupportedMdmType(resourceType)) { + errorMsgs.add("Resource type " + resourceType + " is not supported by MDM. Check your MDM settings"); + } + } + + private String getResourceTypeFromUrl(String url) { + int questionMarkIndex = url.indexOf('?'); + String resourceType; + if (questionMarkIndex == -1) { + resourceType = url; + } else { + resourceType = url.substring(0, questionMarkIndex); + } + return resourceType; + } +} diff --git a/hapi-fhir-storage-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidatorTest.java b/hapi-fhir-storage-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidatorTest.java new file mode 100644 index 00000000000..b7872930f64 --- /dev/null +++ b/hapi-fhir-storage-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/submit/MdmSubmitJobParametersValidatorTest.java @@ -0,0 +1,67 @@ +package ca.uhn.fhir.mdm.batch2.submit; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import ca.uhn.fhir.mdm.api.IMdmSettings; +import ca.uhn.fhir.mdm.rules.json.MdmRulesJson; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + + +@ExtendWith(MockitoExtension.class) +class MdmSubmitJobParametersValidatorTest { + + @InjectMocks + MdmSubmitJobParametersValidator myValidator; + @Mock + private FhirContext myFhirContext; + @Mock + private IMdmSettings myMdmSettings; + @Mock + private MatchUrlService myMatchUrlService; + + @BeforeEach + public void before() { + myFhirContext = FhirContext.forR4Cached(); + myValidator = new MdmSubmitJobParametersValidator(myMdmSettings, myMatchUrlService, myFhirContext); + } + + @Test + public void testUnusedMdmResourceTypesAreNotAccepted() { + when(myMdmSettings.isSupportedMdmType(anyString())).thenReturn(false); + + MdmSubmitJobParameters parameters = new MdmSubmitJobParameters(); + parameters.addUrl("Practitioner?name=foo"); + List errors = myValidator.validate(parameters); + assertThat(errors, hasSize(1)); + assertThat(errors.get(0), is(equalTo("Resource type Practitioner is not supported by MDM. Check your MDM settings"))); + } + + @Test + public void testMissingSearchParameter() { + when(myMdmSettings.isSupportedMdmType(anyString())).thenReturn(true); + when(myMatchUrlService.translateMatchUrl(anyString(), any())).thenThrow(new InvalidRequestException("Can't find death-date!")); + MdmSubmitJobParameters parameters = new MdmSubmitJobParameters(); + parameters.addUrl("Practitioner?death-date=foo"); + List errors = myValidator.validate(parameters); + assertThat(errors, hasSize(1)); + assertThat(errors.get(0), is(equalTo("Invalid request detected: Can't find death-date!"))); + } + +}