From 346e29920ac52458747597a87aa19571679b7cdd Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Fri, 3 Sep 2021 14:58:19 -0400 Subject: [PATCH] Issue 2927 convert mdm clear to spring batch (#2929) * initial roughout of mdm clear batch job * still roughing out classes * finished first draft of reader * most tests passing now * all tests pass. now FIXMEs * FIXMEs done. Time for regression. * fix test * changelog and docs * fix test * pre-review cleanup * version bump * move spring autowire deps for cdr * move spring autowire deps for cdr * finally got beans working phew * rearrange method calls so persistence module can perform clear operation on its own * Update hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md Co-authored-by: Tadgh * review feedback * review feedback * Remove inheritance * fix beans Co-authored-by: Tadgh Co-authored-by: Tadgh --- hapi-deployable-pom/pom.xml | 2 +- hapi-fhir-android/pom.xml | 2 +- hapi-fhir-base/pom.xml | 2 +- hapi-fhir-bom/pom.xml | 4 +- hapi-fhir-cli/hapi-fhir-cli-api/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-app/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml | 2 +- hapi-fhir-cli/pom.xml | 2 +- hapi-fhir-client-okhttp/pom.xml | 2 +- hapi-fhir-client/pom.xml | 2 +- hapi-fhir-converter/pom.xml | 2 +- hapi-fhir-dist/pom.xml | 2 +- hapi-fhir-docs/pom.xml | 2 +- .../5_6_0/2927-mdm-clear-spring-batch.yaml | 3 + .../docs/server_jpa_mdm/mdm_operations.md | 54 +++-- hapi-fhir-jacoco/pom.xml | 2 +- hapi-fhir-jaxrsserver-base/pom.xml | 2 +- hapi-fhir-jpaserver-api/pom.xml | 2 +- hapi-fhir-jpaserver-base/pom.xml | 2 +- .../uhn/fhir/jpa/batch/BatchJobsConfig.java | 9 +- .../fhir/jpa/batch/CommonBatchJobConfig.java | 31 +++ .../batch/job/MultiUrlProcessorJobConfig.java | 57 ----- .../batch/job/PartitionedUrlValidator.java | 3 + .../mdm/MdmBatchJobSubmitterFactoryImpl.java | 15 ++ .../batch/mdm/MdmClearJobSubmitterImpl.java | 84 ++++++++ .../jpa/batch/mdm/job/MdmClearJobConfig.java | 125 +++++++++++ .../jpa/batch/mdm/job/MdmLinkDeleter.java | 85 ++++++++ ...erseCronologicalBatchMdmLinkPidReader.java | 52 +++++ ...BaseReverseCronologicalBatchPidReader.java | 193 +++++++++++++++++ ...CronologicalBatchAllResourcePidReader.java | 4 +- ...rseCronologicalBatchResourcePidReader.java | 183 ++-------------- .../ca/uhn/fhir/jpa/config/BaseConfig.java | 22 +- .../ca/uhn/fhir/jpa/dao/data/IMdmLinkDao.java | 7 +- .../jpa/dao/expunge/DeleteExpungeService.java | 201 ------------------ .../fhir/jpa/dao/mdm/MdmLinkExpandSvc.java | 7 +- .../delete/job/DeleteExpungeJobConfig.java | 27 ++- .../job/ReindexEverythingJobConfig.java | 10 +- .../jpa/reindex/job/ReindexJobConfig.java | 25 ++- .../jpa/bulk/BulkDataExportSvcImplR4Test.java | 3 +- .../fhir/jpa/delete/job/ReindexJobTest.java | 4 +- .../r4/MultitenantBatchOperationR4Test.java | 4 +- .../jpa/provider/r4/SystemProviderR4Test.java | 4 +- hapi-fhir-jpaserver-batch/pom.xml | 2 +- hapi-fhir-jpaserver-cql/pom.xml | 2 +- hapi-fhir-jpaserver-mdm/pom.xml | 2 +- .../jpa/mdm/config/MdmConsumerConfig.java | 10 +- .../jpa/mdm/config/MdmSubmitterConfig.java | 15 +- .../uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java | 43 ---- .../uhn/fhir/jpa/mdm/svc/MdmClearSvcImpl.java | 84 -------- .../jpa/mdm/svc/MdmControllerSvcImpl.java | 21 ++ .../mdm/svc/MdmGoldenResourceDeletingSvc.java | 57 ----- .../fhir/jpa/mdm/svc/MdmSubmitSvcImpl.java | 11 +- .../jpa/mdm/config/BaseTestMdmConfig.java | 11 +- .../fhir/jpa/mdm/provider/BaseLinkR4Test.java | 6 +- .../jpa/mdm/provider/BaseProviderR4Test.java | 36 +++- .../mdm/provider/MdmProviderBatchR4Test.java | 26 +-- .../provider/MdmProviderClearLinkR4Test.java | 22 +- hapi-fhir-jpaserver-migrate/pom.xml | 2 +- hapi-fhir-jpaserver-model/pom.xml | 2 +- hapi-fhir-jpaserver-searchparam/pom.xml | 2 +- hapi-fhir-jpaserver-subscription/pom.xml | 2 +- hapi-fhir-jpaserver-test-utilities/pom.xml | 2 +- hapi-fhir-jpaserver-uhnfhirtest/pom.xml | 2 +- hapi-fhir-server-mdm/pom.xml | 2 +- .../mdm/api/IMdmBatchJobSubmitterFactory.java | 5 + ...ngeSvc.java => IMdmClearJobSubmitter.java} | 24 +-- .../uhn/fhir/mdm/api/IMdmControllerSvc.java | 7 + .../mdm/provider/MdmProviderDstu3Plus.java | 62 +++--- .../fhir/mdm/provider/MdmProviderLoader.java | 11 +- hapi-fhir-server-openapi/pom.xml | 2 +- hapi-fhir-server/pom.xml | 2 +- .../provider/DeleteExpungeProvider.java | 9 +- ...lProcessor.java => MultiUrlProcessor.java} | 12 +- .../server/provider/ProviderConstants.java | 14 +- .../rest/server/provider/ReindexProvider.java | 13 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../hapi-fhir-spring-boot-samples/pom.xml | 2 +- .../hapi-fhir-spring-boot-starter/pom.xml | 2 +- hapi-fhir-spring-boot/pom.xml | 2 +- hapi-fhir-structures-dstu2.1/pom.xml | 2 +- hapi-fhir-structures-dstu2/pom.xml | 2 +- hapi-fhir-structures-dstu3/pom.xml | 2 +- hapi-fhir-structures-hl7org-dstu2/pom.xml | 2 +- hapi-fhir-structures-r4/pom.xml | 2 +- .../rest/server/helper/BatchHelperR4.java | 16 ++ .../server/provider/BatchProviderTest.java | 4 +- hapi-fhir-structures-r5/pom.xml | 2 +- hapi-fhir-test-utilities/pom.xml | 2 +- hapi-fhir-testpage-overlay/pom.xml | 2 +- .../pom.xml | 2 +- hapi-fhir-validation-resources-dstu2/pom.xml | 2 +- hapi-fhir-validation-resources-dstu3/pom.xml | 2 +- hapi-fhir-validation-resources-r4/pom.xml | 2 +- hapi-fhir-validation-resources-r5/pom.xml | 2 +- hapi-fhir-validation/pom.xml | 2 +- hapi-tinder-plugin/pom.xml | 16 +- hapi-tinder-test/pom.xml | 2 +- pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- 104 files changed, 990 insertions(+), 869 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/2927-mdm-clear-spring-batch.yaml delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlProcessorJobConfig.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmBatchJobSubmitterFactoryImpl.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmClearJobSubmitterImpl.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmClearJobConfig.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleter.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/ReverseCronologicalBatchMdmLinkPidReader.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeService.java delete mode 100644 hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmClearSvcImpl.java delete mode 100644 hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmGoldenResourceDeletingSvc.java create mode 100644 hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java rename hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/{IMdmExpungeSvc.java => IMdmClearJobSubmitter.java} (51%) rename hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/{BaseMultiUrlProcessor.java => MultiUrlProcessor.java} (80%) create mode 100644 hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/rest/server/helper/BatchHelperR4.java diff --git a/hapi-deployable-pom/pom.xml b/hapi-deployable-pom/pom.xml index 97a7f698966..266c65a230b 100644 --- a/hapi-deployable-pom/pom.xml +++ b/hapi-deployable-pom/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-android/pom.xml b/hapi-fhir-android/pom.xml index 3fb74827f93..f6390a03ab5 100644 --- a/hapi-fhir-android/pom.xml +++ b/hapi-fhir-android/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/pom.xml b/hapi-fhir-base/pom.xml index a7ac0302705..311fd519e79 100644 --- a/hapi-fhir-base/pom.xml +++ b/hapi-fhir-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-bom/pom.xml b/hapi-fhir-bom/pom.xml index 8299f632b49..0ee5d85b77e 100644 --- a/hapi-fhir-bom/pom.xml +++ b/hapi-fhir-bom/pom.xml @@ -3,14 +3,14 @@ 4.0.0 ca.uhn.hapi.fhir hapi-fhir-bom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT pom HAPI FHIR BOM ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml index d758c2fa826..05da3e3fee3 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml index 6754bf399d6..acc6ac71323 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir-cli - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml index a7c0c5300e3..2b8034bb414 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../../hapi-deployable-pom diff --git a/hapi-fhir-cli/pom.xml b/hapi-fhir-cli/pom.xml index 87195288afe..86dc09e7364 100644 --- a/hapi-fhir-cli/pom.xml +++ b/hapi-fhir-cli/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-client-okhttp/pom.xml b/hapi-fhir-client-okhttp/pom.xml index 13c7f7af8bd..7b1c2cabc2a 100644 --- a/hapi-fhir-client-okhttp/pom.xml +++ b/hapi-fhir-client-okhttp/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-client/pom.xml b/hapi-fhir-client/pom.xml index 7ac45a8c943..10ac2018c17 100644 --- a/hapi-fhir-client/pom.xml +++ b/hapi-fhir-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-converter/pom.xml b/hapi-fhir-converter/pom.xml index 10f8d934c02..df4e808eaff 100644 --- a/hapi-fhir-converter/pom.xml +++ b/hapi-fhir-converter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-dist/pom.xml b/hapi-fhir-dist/pom.xml index 3c20dd131b9..bca1b836071 100644 --- a/hapi-fhir-dist/pom.xml +++ b/hapi-fhir-dist/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-docs/pom.xml b/hapi-fhir-docs/pom.xml index 8057f47b1cc..3d086916574 100644 --- a/hapi-fhir-docs/pom.xml +++ b/hapi-fhir-docs/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/2927-mdm-clear-spring-batch.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/2927-mdm-clear-spring-batch.yaml new file mode 100644 index 00000000000..518e39472ac --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_6_0/2927-mdm-clear-spring-batch.yaml @@ -0,0 +1,3 @@ +--- +type: change +title: "The $mdm-clear operation has been changed to use Spring Batch." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md index b946438ebcb..7679c532977 100644 --- a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_mdm/mdm_operations.md @@ -567,11 +567,15 @@ Note that the request goes to the root of the FHIR server, and not the `Organiza ## Clearing MDM Links -The `$mdm-clear` operation is used to batch-delete MDM links and related Golden Resources from the database. This operation is meant to be used during the rules-tuning phase of the MDM implementation so that you can quickly test your ruleset. It permits the user to reset the state of their MDM system without manual deletion of all related links and Golden Resources. +The `$mdm-clear` operation is used to batch-delete MDM links and related Golden Resources from the database. This +operation is intended to be used during the rules-tuning phase of the MDM implementation so that you can quickly test +your ruleset. It permits the user to reset the state of their MDM system without manual deletion of all related links +and Golden Resources. -After the operation is complete, all targeted MDM links are removed from the system, and their related Golden Resources are deleted and expunged from the server. +After the operation is complete, all targeted MDM links are removed from the system, and their related Golden Resources +are deleted and expunged from the server. -This operation takes a single optional Parameter. +This operation takes two optional Parameters. @@ -584,11 +588,21 @@ This operation takes a single optional Parameter. - + + + + + + + @@ -598,33 +612,27 @@ This operation takes a single optional Parameter. Use an HTTP POST to the following URL to invoke this operation: -```url -http://example.com/$mdm-clear -``` +```http +POST /$mdm-clear +Content-Type: application/fhir+json -The following request body could be used: - -```json { "resourceType": "Parameters", "parameter": [ { - "name": "sourceType", + "name": "resourceType", "valueString": "Patient" + }, { + "name": "resourceType", + "valueString": "Practitioner" + }, { + "name": "batchSize", + "valueDecimal": 1000 } ] } ``` -This operation returns the number of MDM links that were cleared. The following is a sample response: - -```json -{ - "resourceType": "Parameters", - "parameter": [ { - "name": "reset", - "valueDecimal": 5 - } ] -} -``` +This operation returns the job execution id of the Spring Batch job that will be run to remove all the links and their +golden resources. ## Batch-creating MDM Links diff --git a/hapi-fhir-jacoco/pom.xml b/hapi-fhir-jacoco/pom.xml index ba7399f2148..3001c607337 100644 --- a/hapi-fhir-jacoco/pom.xml +++ b/hapi-fhir-jacoco/pom.xml @@ -11,7 +11,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jaxrsserver-base/pom.xml b/hapi-fhir-jaxrsserver-base/pom.xml index 9a6548cb58a..ed82d9af335 100644 --- a/hapi-fhir-jaxrsserver-base/pom.xml +++ b/hapi-fhir-jaxrsserver-base/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-api/pom.xml b/hapi-fhir-jpaserver-api/pom.xml index 7e5767c9039..f763646e3f4 100644 --- a/hapi-fhir-jpaserver-api/pom.xml +++ b/hapi-fhir-jpaserver-api/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index 9a5909c79de..8fb7260401d 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java index 4314a68c8b5..9e96243d45c 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.batch; * #L% */ +import ca.uhn.fhir.jpa.batch.mdm.job.MdmClearJobConfig; import ca.uhn.fhir.jpa.bulk.export.job.BulkExportJobConfig; import ca.uhn.fhir.jpa.bulk.imprt.job.BulkImportJobConfig; import ca.uhn.fhir.jpa.delete.job.DeleteExpungeJobConfig; @@ -40,7 +41,8 @@ import java.util.Set; BulkImportJobConfig.class, DeleteExpungeJobConfig.class, ReindexJobConfig.class, - ReindexEverythingJobConfig.class + ReindexEverythingJobConfig.class, + MdmClearJobConfig.class }) public class BatchJobsConfig { @@ -94,4 +96,9 @@ public class BatchJobsConfig { */ public static final String REINDEX_EVERYTHING_JOB_NAME = "reindexEverythingJob"; + /** + * MDM Clear + */ + public static final String MDM_CLEAR_JOB_NAME = "mdmClearJob"; + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java index a681e7e6bbb..e4eaf574d52 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java @@ -20,15 +20,46 @@ package ca.uhn.fhir.jpa.batch; * #L% */ +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; +import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; import ca.uhn.fhir.jpa.batch.processor.GoldenResourceAnnotatingProcessor; import ca.uhn.fhir.jpa.batch.processor.PidToIBaseResourceProcessor; +import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; +import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; import ca.uhn.fhir.jpa.reindex.job.ReindexWriter; +import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import org.springframework.batch.core.JobParametersValidator; import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration public class CommonBatchJobConfig { + public static final int MINUTES_IN_FUTURE_TO_PROCESS_FROM = 1; + + @Bean + public MultiUrlJobParameterValidator multiUrlProcessorParameterValidator(MatchUrlService theMatchUrlService, DaoRegistry theDaoRegistry) { + return new MultiUrlJobParameterValidator(theMatchUrlService, theDaoRegistry); + } + + @Bean + @StepScope + public SqlExecutorWriter sqlExecutorWriter() { + return new SqlExecutorWriter(); + } + + @Bean + @StepScope + public PidReaderCounterListener pidCountRecorderListener() { + return new PidReaderCounterListener(); + } + + @Bean + @StepScope + public ReverseCronologicalBatchResourcePidReader reverseCronologicalBatchResourcePidReader() { + return new ReverseCronologicalBatchResourcePidReader(); + } @Bean @StepScope diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlProcessorJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlProcessorJobConfig.java deleted file mode 100644 index c47f01b59d1..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlProcessorJobConfig.java +++ /dev/null @@ -1,57 +0,0 @@ -package ca.uhn.fhir.jpa.batch.job; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2021 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; -import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; -import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; -import org.springframework.batch.core.JobParametersValidator; -import org.springframework.batch.core.configuration.annotation.StepScope; -import org.springframework.context.annotation.Bean; - -public class MultiUrlProcessorJobConfig { - public static final int MINUTES_IN_FUTURE_TO_PROCESS_FROM = 1; - - @Bean - public JobParametersValidator multiUrlProcessorParameterValidator(MatchUrlService theMatchUrlService, DaoRegistry theDaoRegistry) { - return new MultiUrlJobParameterValidator(theMatchUrlService, theDaoRegistry); - } - - @Bean - @StepScope - public SqlExecutorWriter sqlExecutorWriter() { - return new SqlExecutorWriter(); - } - - @Bean - @StepScope - public PidReaderCounterListener pidCountRecorderListener() { - return new PidReaderCounterListener(); - } - - @Bean - @StepScope - public ReverseCronologicalBatchResourcePidReader reverseCronologicalBatchResourcePidReader() { - return new ReverseCronologicalBatchResourcePidReader(); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java index 80f69936f3e..0620b296e71 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java @@ -43,6 +43,9 @@ public class PartitionedUrlValidator { @Autowired FhirContext myFhirContext; + public PartitionedUrlValidator() { + } + /** * This method will throw an exception if the user is not allowed to access the requested resource type on the partition determined by the request */ diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmBatchJobSubmitterFactoryImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmBatchJobSubmitterFactoryImpl.java new file mode 100644 index 00000000000..e31df97cc3c --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmBatchJobSubmitterFactoryImpl.java @@ -0,0 +1,15 @@ +package ca.uhn.fhir.jpa.batch.mdm; + +import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory; +import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; +import org.springframework.beans.factory.annotation.Autowired; + +public class MdmBatchJobSubmitterFactoryImpl implements IMdmBatchJobSubmitterFactory { + @Autowired + IMdmClearJobSubmitter myMdmClearJobSubmitter; + + @Override + public IMdmClearJobSubmitter getClearJobSubmitter() { + return myMdmClearJobSubmitter; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmClearJobSubmitterImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmClearJobSubmitterImpl.java new file mode 100644 index 00000000000..90bee86b963 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/MdmClearJobSubmitterImpl.java @@ -0,0 +1,84 @@ +package ca.uhn.fhir.jpa.batch.mdm; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2021 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.interceptor.api.HookParams; +import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; +import ca.uhn.fhir.interceptor.api.Pointcut; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.batch.BatchJobsConfig; +import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; +import ca.uhn.fhir.jpa.batch.job.PartitionedUrlValidator; +import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; +import ca.uhn.fhir.jpa.batch.mdm.job.ReverseCronologicalBatchMdmLinkPidReader; +import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.server.exceptions.ForbiddenOperationException; +import ca.uhn.fhir.rest.server.provider.ProviderConstants; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersInvalidException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; + +import javax.transaction.Transactional; +import java.util.List; + +public class MdmClearJobSubmitterImpl implements IMdmClearJobSubmitter { + @Autowired + DaoConfig myDaoConfig; + @Autowired + PartitionedUrlValidator myPartitionedUrlValidator; + @Autowired + IInterceptorBroadcaster myInterceptorBroadcaster; + @Autowired + private IBatchJobSubmitter myBatchJobSubmitter; + @Autowired + @Qualifier(BatchJobsConfig.MDM_CLEAR_JOB_NAME) + private Job myMdmClearJob; + + @Override + @Transactional(Transactional.TxType.NEVER) + public JobExecution submitJob(Integer theBatchSize, List theUrls, RequestDetails theRequest) throws JobParametersInvalidException { + if (theBatchSize == null) { + theBatchSize = myDaoConfig.getExpungeBatchSize(); + } + if (!myDaoConfig.canDeleteExpunge()) { + throw new ForbiddenOperationException("Delete Expunge not allowed: " + myDaoConfig.cannotDeleteExpungeReason()); + } + + RequestListJson requestListJson = myPartitionedUrlValidator.buildRequestListJson(theRequest, theUrls); + + for (String url : theUrls) { + HookParams params = new HookParams() + .add(RequestDetails.class, theRequest) + .addIfMatchesType(ServletRequestDetails.class, theRequest) + .add(String.class, url); + CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PRE_DELETE_EXPUNGE, params); + } + + JobParameters jobParameters = ReverseCronologicalBatchMdmLinkPidReader.buildJobParameters(ProviderConstants.OPERATION_MDM_CLEAR, theBatchSize, requestListJson); + return myBatchJobSubmitter.runJob(myMdmClearJob, jobParameters); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmClearJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmClearJobConfig.java new file mode 100644 index 00000000000..b2786f7cc84 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmClearJobConfig.java @@ -0,0 +1,125 @@ +package ca.uhn.fhir.jpa.batch.mdm.job; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2021 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; +import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; +import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; +import ca.uhn.fhir.jpa.delete.job.DeleteExpungeProcessor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.listener.ExecutionContextPromotionListener; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.support.CompositeItemProcessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; + +import java.util.ArrayList; +import java.util.List; + +import static ca.uhn.fhir.jpa.batch.BatchJobsConfig.MDM_CLEAR_JOB_NAME; + +/** + * Spring batch Job configuration file. Contains all necessary plumbing to run a + * $mdm-clear job. + */ +@Configuration +public class MdmClearJobConfig { + public static final String MDM_CLEAR_RESOURCE_LIST_STEP_NAME = "mdm-clear-resource-list-step"; + + @Autowired + private StepBuilderFactory myStepBuilderFactory; + @Autowired + private JobBuilderFactory myJobBuilderFactory; + @Autowired + private DeleteExpungeProcessor myDeleteExpungeProcessor; + + @Autowired + @Qualifier("deleteExpungePromotionListener") + private ExecutionContextPromotionListener myDeleteExpungePromotionListener; + + @Autowired + private MultiUrlJobParameterValidator myMultiUrlProcessorParameterValidator; + + @Autowired + private PidReaderCounterListener myPidCountRecorderListener; + + @Autowired + private SqlExecutorWriter mySqlExecutorWriter; + + @Bean(name = MDM_CLEAR_JOB_NAME) + @Lazy + public Job mdmClearJob() { + return myJobBuilderFactory.get(MDM_CLEAR_JOB_NAME) + .validator(myMultiUrlProcessorParameterValidator) + .start(mdmClearUrlListStep()) + .build(); + } + + @Bean + public Step mdmClearUrlListStep() { + return myStepBuilderFactory.get(MDM_CLEAR_RESOURCE_LIST_STEP_NAME) + ., List>chunk(1) + .reader(reverseCronologicalBatchMdmLinkPidReader()) + .processor(deleteThenExpungeCompositeProcessor()) + .writer(mySqlExecutorWriter) + .listener(myPidCountRecorderListener) + .listener(myDeleteExpungePromotionListener) + .build(); + } + + @Bean + @StepScope + public ItemProcessor, List> deleteThenExpungeCompositeProcessor() { + CompositeItemProcessor, List> compositeProcessor = new CompositeItemProcessor<>(); + List itemProcessors = new ArrayList<>(); + itemProcessors.add(mdmLinkDeleter()); + itemProcessors.add(myDeleteExpungeProcessor); + compositeProcessor.setDelegates(itemProcessors); + return compositeProcessor; + } + + @Bean + @StepScope + public ReverseCronologicalBatchMdmLinkPidReader reverseCronologicalBatchMdmLinkPidReader() { + return new ReverseCronologicalBatchMdmLinkPidReader(); + } + + @Bean + public MdmLinkDeleter mdmLinkDeleter() { + return new MdmLinkDeleter(); + } + + @Bean + public ExecutionContextPromotionListener mdmClearPromotionListener() { + ExecutionContextPromotionListener listener = new ExecutionContextPromotionListener(); + + listener.setKeys(new String[]{PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED}); + + return listener; + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleter.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleter.java new file mode 100644 index 00000000000..e43cc5b07a4 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleter.java @@ -0,0 +1,85 @@ +package ca.uhn.fhir.jpa.batch.mdm.job; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2021 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.dao.data.IMdmLinkDao; +import ca.uhn.fhir.jpa.dao.expunge.PartitionRunner; +import ca.uhn.fhir.jpa.entity.MdmLink; +import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.SliceImpl; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; + +/** + * Take MdmLink pids in and output golden resource pids out + */ + +public class MdmLinkDeleter implements ItemProcessor, List> { + public static final String PROCESS_NAME = "MdmClear"; + public static final String THREAD_PREFIX = "mdmClear"; + private static final Logger ourLog = LoggerFactory.getLogger(MdmLinkDeleter.class); + @Autowired + protected PlatformTransactionManager myTxManager; + @Autowired + IMdmLinkDao myMdmLinkDao; + @Autowired + DaoConfig myDaoConfig; + + @Override + public List process(List thePidList) throws Exception { + ConcurrentLinkedQueue goldenPidAggregator = new ConcurrentLinkedQueue<>(); + PartitionRunner partitionRunner = new PartitionRunner(PROCESS_NAME, THREAD_PREFIX, myDaoConfig.getReindexBatchSize(), myDaoConfig.getReindexThreadCount()); + partitionRunner.runInPartitionedThreads(new SliceImpl<>(thePidList), pids -> removeLinks(thePidList, goldenPidAggregator)); + return new ArrayList<>(goldenPidAggregator); + } + + private void removeLinks(List pidList, ConcurrentLinkedQueue theGoldenPidAggregator) { + TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); + + txTemplate.executeWithoutResult(t -> theGoldenPidAggregator.addAll(deleteMdmLinksAndReturnGoldenResourcePids(pidList))); + } + + public List deleteMdmLinksAndReturnGoldenResourcePids(List thePids) { + List links = myMdmLinkDao.findAllById(thePids); + Set goldenResources = links.stream().map(MdmLink::getGoldenResourcePid).collect(Collectors.toSet()); + //TODO GGG this is probably invalid... we are essentially looking for GOLDEN -> GOLDEN links, which are either POSSIBLE_DUPLICATE + //and REDIRECT + goldenResources.addAll(links.stream() + .filter(link -> link.getMatchResult().equals(MdmMatchResultEnum.REDIRECT) + || link.getMatchResult().equals(MdmMatchResultEnum.POSSIBLE_DUPLICATE)) + .map(MdmLink::getSourcePid).collect(Collectors.toSet())); + ourLog.info("Deleting {} MDM link records...", links.size()); + myMdmLinkDao.deleteAll(links); + ourLog.info("{} MDM link records deleted", links.size()); + return new ArrayList<>(goldenResources); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/ReverseCronologicalBatchMdmLinkPidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/ReverseCronologicalBatchMdmLinkPidReader.java new file mode 100644 index 00000000000..8ad752f96fe --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/job/ReverseCronologicalBatchMdmLinkPidReader.java @@ -0,0 +1,52 @@ +package ca.uhn.fhir.jpa.batch.mdm.job; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2021 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.batch.reader.BaseReverseCronologicalBatchPidReader; +import ca.uhn.fhir.jpa.dao.data.IMdmLinkDao; +import ca.uhn.fhir.jpa.searchparam.ResourceSearch; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; + +import java.util.HashSet; +import java.util.Set; + +/** + * This is the same as the parent class, except it operates on MdmLink entities instead of resource entities + */ +public class ReverseCronologicalBatchMdmLinkPidReader extends BaseReverseCronologicalBatchPidReader { + @Autowired + IMdmLinkDao myMdmLinkDao; + + @Override + protected Set getNextPidBatch(ResourceSearch resourceSearch) { + String resourceName = resourceSearch.getResourceName(); + Pageable pageable = PageRequest.of(0, getBatchSize()); + //Expand out the list to handle the REDIRECT/POSSIBLE DUPLICATE ones. + return new HashSet<>(myMdmLinkDao.findPidByResourceNameAndThreshold(resourceName, getCurrentHighThreshold(), pageable)); + } + + @Override + protected void setDateFromPidFunction(ResourceSearch resourceSearch) { + setDateExtractorFunction(pid -> myMdmLinkDao.findById(pid).get().getCreated()); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java new file mode 100644 index 00000000000..d682728119c --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java @@ -0,0 +1,193 @@ +package ca.uhn.fhir.jpa.batch.reader; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.batch.CommonBatchJobConfig; +import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; +import ca.uhn.fhir.jpa.batch.job.model.PartitionedUrl; +import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; +import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import ca.uhn.fhir.jpa.searchparam.ResourceSearch; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.SortOrderEnum; +import ca.uhn.fhir.rest.api.SortSpec; +import ca.uhn.fhir.rest.param.DateRangeParam; +import org.apache.commons.lang3.time.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.batch.core.JobParameter; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemStream; +import org.springframework.batch.item.ItemStreamException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** + * This Spring Batch reader takes 4 parameters: + * {@link #JOB_PARAM_REQUEST_LIST}: A list of URLs to search for along with the partitions those searches should be performed on + * {@link #JOB_PARAM_BATCH_SIZE}: The number of resources to return with each search. If ommitted, {@link DaoConfig#getExpungeBatchSize} will be used. + * {@link #JOB_PARAM_START_TIME}: The latest timestamp of entities to search for + *

+ * The reader will return at most {@link #JOB_PARAM_BATCH_SIZE} pids every time it is called, or null + * once no more matching entities are available. It returns the resources in reverse chronological order + * and stores where it's at in the Spring Batch execution context with the key {@link #CURRENT_THRESHOLD_HIGH} + * appended with "." and the index number of the url list item it has gotten up to. This is to permit + * restarting jobs that use this reader so it can pick up where it left off. + */ +public abstract class BaseReverseCronologicalBatchPidReader implements ItemReader>, ItemStream { + public static final String JOB_PARAM_REQUEST_LIST = "url-list"; + public static final String JOB_PARAM_BATCH_SIZE = "batch-size"; + public static final String JOB_PARAM_START_TIME = "start-time"; + public static final String CURRENT_URL_INDEX = "current.url-index"; + public static final String CURRENT_THRESHOLD_HIGH = "current.threshold-high"; + private static final Logger ourLog = LoggerFactory.getLogger(ReverseCronologicalBatchResourcePidReader.class); + private final BatchDateThresholdUpdater myBatchDateThresholdUpdater = new BatchDateThresholdUpdater(); + private final Map myThresholdHighByUrlIndex = new HashMap<>(); + private final Map> myAlreadyProcessedPidsWithHighDate = new HashMap<>(); + @Autowired + private FhirContext myFhirContext; + @Autowired + private MatchUrlService myMatchUrlService; + private List myPartitionedUrls; + private Integer myBatchSize; + private int myUrlIndex = 0; + private Date myStartTime; + + private static String highKey(int theIndex) { + return CURRENT_THRESHOLD_HIGH + "." + theIndex; + } + + @Nonnull + public static JobParameters buildJobParameters(String theOperationName, Integer theBatchSize, RequestListJson theRequestListJson) { + Map map = new HashMap<>(); + map.put(MultiUrlJobParameterValidator.JOB_PARAM_OPERATION_NAME, new JobParameter(theOperationName)); + map.put(ReverseCronologicalBatchResourcePidReader.JOB_PARAM_REQUEST_LIST, new JobParameter(theRequestListJson.toJson())); + map.put(ReverseCronologicalBatchResourcePidReader.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), CommonBatchJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); + if (theBatchSize != null) { + map.put(ReverseCronologicalBatchResourcePidReader.JOB_PARAM_BATCH_SIZE, new JobParameter(theBatchSize.longValue())); + } + JobParameters parameters = new JobParameters(map); + return parameters; + } + + @Autowired + public void setRequestListJson(@Value("#{jobParameters['" + JOB_PARAM_REQUEST_LIST + "']}") String theRequestListJson) { + RequestListJson requestListJson = RequestListJson.fromJson(theRequestListJson); + myPartitionedUrls = requestListJson.getPartitionedUrls(); + } + + @Autowired + public void setStartTime(@Value("#{jobParameters['" + JOB_PARAM_START_TIME + "']}") Date theStartTime) { + myStartTime = theStartTime; + } + + @Override + public List read() throws Exception { + while (myUrlIndex < myPartitionedUrls.size()) { + List nextBatch = getNextBatch(); + if (nextBatch.isEmpty()) { + ++myUrlIndex; + continue; + } + + return nextBatch; + } + return null; + } + + protected List getNextBatch() { + RequestPartitionId requestPartitionId = myPartitionedUrls.get(myUrlIndex).getRequestPartitionId(); + ResourceSearch resourceSearch = myMatchUrlService.getResourceSearch(myPartitionedUrls.get(myUrlIndex).getUrl(), requestPartitionId); + myAlreadyProcessedPidsWithHighDate.putIfAbsent(myUrlIndex, new HashSet<>()); + Set newPids = getNextPidBatch(resourceSearch); + + if (ourLog.isDebugEnabled()) { + ourLog.debug("Search for {}{} returned {} results", resourceSearch.getResourceName(), resourceSearch.getSearchParameterMap().toNormalizedQueryString(myFhirContext), newPids.size()); + ourLog.debug("Results: {}", newPids); + } + + setDateFromPidFunction(resourceSearch); + + List retval = new ArrayList<>(newPids); + Date newThreshold = myBatchDateThresholdUpdater.updateThresholdAndCache(getCurrentHighThreshold(), myAlreadyProcessedPidsWithHighDate.get(myUrlIndex), retval); + myThresholdHighByUrlIndex.put(myUrlIndex, newThreshold); + + return retval; + } + + protected Date getCurrentHighThreshold() { + return myThresholdHighByUrlIndex.get(myUrlIndex); + } + + protected void setDateExtractorFunction(Function theDateExtractorFunction) { + myBatchDateThresholdUpdater.setDateFromPid(theDateExtractorFunction); + } + + protected void addDateCountAndSortToSearch(ResourceSearch resourceSearch) { + SearchParameterMap map = resourceSearch.getSearchParameterMap(); + map.setLastUpdated(new DateRangeParam().setUpperBoundInclusive(getCurrentHighThreshold())); + map.setLoadSynchronousUpTo(myBatchSize); + map.setSort(new SortSpec(Constants.PARAM_LASTUPDATED, SortOrderEnum.DESC)); + } + + @Override + public void open(ExecutionContext executionContext) throws ItemStreamException { + if (executionContext.containsKey(CURRENT_URL_INDEX)) { + myUrlIndex = new Long(executionContext.getLong(CURRENT_URL_INDEX)).intValue(); + } + for (int index = 0; index < myPartitionedUrls.size(); ++index) { + String key = highKey(index); + if (executionContext.containsKey(key)) { + myThresholdHighByUrlIndex.put(index, new Date(executionContext.getLong(key))); + } else { + myThresholdHighByUrlIndex.put(index, myStartTime); + } + } + } + + @Override + public void update(ExecutionContext executionContext) throws ItemStreamException { + executionContext.putLong(CURRENT_URL_INDEX, myUrlIndex); + for (int index = 0; index < myPartitionedUrls.size(); ++index) { + Date date = myThresholdHighByUrlIndex.get(index); + if (date != null) { + executionContext.putLong(highKey(index), date.getTime()); + } + } + } + + @Override + public void close() throws ItemStreamException { + } + + protected Integer getBatchSize() { + return myBatchSize; + } + + @Autowired + public void setBatchSize(@Value("#{jobParameters['" + JOB_PARAM_BATCH_SIZE + "']}") Integer theBatchSize) { + myBatchSize = theBatchSize; + } + + protected Set getAlreadySeenPids() { + return myAlreadyProcessedPidsWithHighDate.get(myUrlIndex); + } + + protected abstract Set getNextPidBatch(ResourceSearch resourceSearch); + + protected abstract void setDateFromPidFunction(ResourceSearch resourceSearch); +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java index 6eeed5db0f6..a704d59e2ea 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java @@ -22,7 +22,7 @@ package ca.uhn.fhir.jpa.batch.reader; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.job.MultiUrlProcessorJobConfig; +import ca.uhn.fhir.jpa.batch.CommonBatchJobConfig; import ca.uhn.fhir.jpa.dao.data.IResourceTableDao; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import com.fasterxml.jackson.core.JsonProcessingException; @@ -93,7 +93,7 @@ public class CronologicalBatchAllResourcePidReader implements ItemReader map = new HashMap<>(); map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_REQUEST_PARTITION, new JobParameter(theRequestPartitionId.toJson())); - map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), MultiUrlProcessorJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); + map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), CommonBatchJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); if (theBatchSize != null) { map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_BATCH_SIZE, new JobParameter(theBatchSize.longValue())); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java index c27e2089b4f..ba840600337 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java @@ -20,209 +20,52 @@ package ca.uhn.fhir.jpa.batch.reader; * #L% */ -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; -import ca.uhn.fhir.jpa.batch.job.MultiUrlProcessorJobConfig; -import ca.uhn.fhir.jpa.batch.job.model.PartitionedUrl; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; import ca.uhn.fhir.jpa.dao.IResultIterator; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.api.Constants; -import ca.uhn.fhir.rest.api.SortOrderEnum; -import ca.uhn.fhir.rest.api.SortSpec; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; -import ca.uhn.fhir.rest.param.DateRangeParam; -import org.apache.commons.lang3.time.DateUtils; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.core.JobParameter; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemStream; -import org.springframework.batch.item.ItemStreamException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import javax.annotation.Nonnull; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -/** - * This Spring Batch reader takes 4 parameters: - * {@link #JOB_PARAM_REQUEST_LIST}: A list of URLs to search for along with the partitions those searches should be performed on - * {@link #JOB_PARAM_BATCH_SIZE}: The number of resources to return with each search. If ommitted, {@link DaoConfig#getExpungeBatchSize} will be used. - * {@link #JOB_PARAM_START_TIME}: The latest timestamp of resources to search for - *

- * The reader will return at most {@link #JOB_PARAM_BATCH_SIZE} pids every time it is called, or null - * once no more matching resources are available. It returns the resources in reverse chronological order - * and stores where it's at in the Spring Batch execution context with the key {@link #CURRENT_THRESHOLD_HIGH} - * appended with "." and the index number of the url list item it has gotten up to. This is to permit - * restarting jobs that use this reader so it can pick up where it left off. - */ -public class ReverseCronologicalBatchResourcePidReader implements ItemReader>, ItemStream { - private static final Logger ourLog = LoggerFactory.getLogger(ReverseCronologicalBatchResourcePidReader.class); - public static final String JOB_PARAM_REQUEST_LIST = "url-list"; - public static final String JOB_PARAM_BATCH_SIZE = "batch-size"; - public static final String JOB_PARAM_START_TIME = "start-time"; - - public static final String CURRENT_URL_INDEX = "current.url-index"; - public static final String CURRENT_THRESHOLD_HIGH = "current.threshold-high"; - - @Autowired - private FhirContext myFhirContext; - @Autowired - private MatchUrlService myMatchUrlService; +public class ReverseCronologicalBatchResourcePidReader extends BaseReverseCronologicalBatchPidReader { @Autowired private DaoRegistry myDaoRegistry; @Autowired private BatchResourceSearcher myBatchResourceSearcher; - private final BatchDateThresholdUpdater myBatchDateThresholdUpdater = new BatchDateThresholdUpdater(); - - private List myPartitionedUrls; - private Integer myBatchSize; - private final Map myThresholdHighByUrlIndex = new HashMap<>(); - private final Map> myAlreadyProcessedPidsWithHighDate = new HashMap<>(); - - private int myUrlIndex = 0; - private Date myStartTime; - - @Autowired - public void setRequestListJson(@Value("#{jobParameters['" + JOB_PARAM_REQUEST_LIST + "']}") String theRequestListJson) { - RequestListJson requestListJson = RequestListJson.fromJson(theRequestListJson); - myPartitionedUrls = requestListJson.getPartitionedUrls(); - } - - @Autowired - public void setBatchSize(@Value("#{jobParameters['" + JOB_PARAM_BATCH_SIZE + "']}") Integer theBatchSize) { - myBatchSize = theBatchSize; - } - - @Autowired - public void setStartTime(@Value("#{jobParameters['" + JOB_PARAM_START_TIME + "']}") Date theStartTime) { - myStartTime = theStartTime; - } - @Override - public List read() throws Exception { - while (myUrlIndex < myPartitionedUrls.size()) { - List nextBatch = getNextBatch(); - if (nextBatch.isEmpty()) { - ++myUrlIndex; - continue; - } - - return nextBatch; - } - return null; - } - - private List getNextBatch() { - RequestPartitionId requestPartitionId = myPartitionedUrls.get(myUrlIndex).getRequestPartitionId(); - ResourceSearch resourceSearch = myMatchUrlService.getResourceSearch(myPartitionedUrls.get(myUrlIndex).getUrl(), requestPartitionId); + protected Set getNextPidBatch(ResourceSearch resourceSearch) { + Set retval = new LinkedHashSet<>(); addDateCountAndSortToSearch(resourceSearch); // Perform the search - IResultIterator resultIter = myBatchResourceSearcher.performSearch(resourceSearch, myBatchSize); - Set newPids = new LinkedHashSet<>(); - Set alreadySeenPids = myAlreadyProcessedPidsWithHighDate.computeIfAbsent(myUrlIndex, i -> new HashSet<>()); + Integer batchSize = getBatchSize(); + IResultIterator resultIter = myBatchResourceSearcher.performSearch(resourceSearch, batchSize); + Set alreadySeenPids = getAlreadySeenPids(); do { - List pids = resultIter.getNextResultBatch(myBatchSize).stream().map(ResourcePersistentId::getIdAsLong).collect(Collectors.toList()); - newPids.addAll(pids); - newPids.removeAll(alreadySeenPids); - } while (newPids.size() < myBatchSize && resultIter.hasNext()); - - if (ourLog.isDebugEnabled()) { - ourLog.debug("Search for {}{} returned {} results", resourceSearch.getResourceName(), resourceSearch.getSearchParameterMap().toNormalizedQueryString(myFhirContext), newPids.size()); - ourLog.debug("Results: {}", newPids); - } - - setDateFromPidFunction(resourceSearch); - - List retval = new ArrayList<>(newPids); - Date newThreshold = myBatchDateThresholdUpdater.updateThresholdAndCache(myThresholdHighByUrlIndex.get(myUrlIndex), myAlreadyProcessedPidsWithHighDate.get(myUrlIndex), retval); - myThresholdHighByUrlIndex.put(myUrlIndex, newThreshold); + List pids = resultIter.getNextResultBatch(batchSize).stream().map(ResourcePersistentId::getIdAsLong).collect(Collectors.toList()); + retval.addAll(pids); + retval.removeAll(alreadySeenPids); + } while (retval.size() < batchSize && resultIter.hasNext()); return retval; } - private void setDateFromPidFunction(ResourceSearch resourceSearch) { + @Override + protected void setDateFromPidFunction(ResourceSearch resourceSearch) { final IFhirResourceDao dao = myDaoRegistry.getResourceDao(resourceSearch.getResourceName()); - myBatchDateThresholdUpdater.setDateFromPid(pid -> { + setDateExtractorFunction(pid -> { IBaseResource oldestResource = dao.readByPid(new ResourcePersistentId(pid)); return oldestResource.getMeta().getLastUpdated(); }); } - - private void addDateCountAndSortToSearch(ResourceSearch resourceSearch) { - SearchParameterMap map = resourceSearch.getSearchParameterMap(); - map.setLastUpdated(new DateRangeParam().setUpperBoundInclusive(myThresholdHighByUrlIndex.get(myUrlIndex))); - map.setLoadSynchronousUpTo(myBatchSize); - map.setSort(new SortSpec(Constants.PARAM_LASTUPDATED, SortOrderEnum.DESC)); - } - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - if (executionContext.containsKey(CURRENT_URL_INDEX)) { - myUrlIndex = new Long(executionContext.getLong(CURRENT_URL_INDEX)).intValue(); - } - for (int index = 0; index < myPartitionedUrls.size(); ++index) { - String key = highKey(index); - if (executionContext.containsKey(key)) { - myThresholdHighByUrlIndex.put(index, new Date(executionContext.getLong(key))); - } else { - myThresholdHighByUrlIndex.put(index, myStartTime); - } - } - } - - private static String highKey(int theIndex) { - return CURRENT_THRESHOLD_HIGH + "." + theIndex; - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - executionContext.putLong(CURRENT_URL_INDEX, myUrlIndex); - for (int index = 0; index < myPartitionedUrls.size(); ++index) { - Date date = myThresholdHighByUrlIndex.get(index); - if (date != null) { - executionContext.putLong(highKey(index), date.getTime()); - } - } - } - - @Override - public void close() throws ItemStreamException { - } - - @Nonnull - public static JobParameters buildJobParameters(String theOperationName, Integer theBatchSize, RequestListJson theRequestListJson) { - Map map = new HashMap<>(); - map.put(MultiUrlJobParameterValidator.JOB_PARAM_OPERATION_NAME, new JobParameter(theOperationName)); - map.put(ReverseCronologicalBatchResourcePidReader.JOB_PARAM_REQUEST_LIST, new JobParameter(theRequestListJson.toJson())); - map.put(ReverseCronologicalBatchResourcePidReader.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), MultiUrlProcessorJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); - if (theBatchSize != null) { - map.put(ReverseCronologicalBatchResourcePidReader.JOB_PARAM_BATCH_SIZE, new JobParameter(theBatchSize.longValue())); - } - JobParameters parameters = new JobParameters(map); - return parameters; - } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java index 2aebff2b539..dfc424f2a77 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/BaseConfig.java @@ -16,6 +16,8 @@ import ca.uhn.fhir.jpa.batch.BatchJobsConfig; import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; import ca.uhn.fhir.jpa.batch.config.NonPersistedBatchConfigurer; import ca.uhn.fhir.jpa.batch.job.PartitionedUrlValidator; +import ca.uhn.fhir.jpa.batch.mdm.MdmBatchJobSubmitterFactoryImpl; +import ca.uhn.fhir.jpa.batch.mdm.MdmClearJobSubmitterImpl; import ca.uhn.fhir.jpa.batch.reader.BatchResourceSearcher; import ca.uhn.fhir.jpa.batch.svc.BatchJobSubmitterImpl; import ca.uhn.fhir.jpa.binstore.BinaryAccessProvider; @@ -35,7 +37,6 @@ import ca.uhn.fhir.jpa.dao.LegacySearchBuilder; import ca.uhn.fhir.jpa.dao.MatchResourceUrlService; import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; import ca.uhn.fhir.jpa.dao.TransactionProcessor; -import ca.uhn.fhir.jpa.dao.expunge.DeleteExpungeService; import ca.uhn.fhir.jpa.dao.expunge.ExpungeEverythingService; import ca.uhn.fhir.jpa.dao.expunge.ExpungeOperation; import ca.uhn.fhir.jpa.dao.expunge.ExpungeService; @@ -136,6 +137,8 @@ import ca.uhn.fhir.jpa.term.api.ITermConceptMappingSvc; import ca.uhn.fhir.jpa.util.MemoryCacheService; import ca.uhn.fhir.jpa.validation.JpaResourceLoader; import ca.uhn.fhir.jpa.validation.ValidationSettings; +import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory; +import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; import ca.uhn.fhir.rest.api.server.storage.IReindexJobSubmitter; @@ -515,10 +518,20 @@ public abstract class BaseConfig { } @Bean - public MdmLinkExpandSvc myMdmLinkExpandSvc() { + public MdmLinkExpandSvc mdmLinkExpandSvc() { return new MdmLinkExpandSvc(); } + @Bean + IMdmBatchJobSubmitterFactory mdmBatchJobSubmitterFactory() { + return new MdmBatchJobSubmitterFactoryImpl(); + } + + @Bean + IMdmClearJobSubmitter mdmClearJobSubmitter() { + return new MdmClearJobSubmitterImpl(); + } + @Bean @Lazy public TerminologyUploaderProvider terminologyUploaderProvider() { @@ -891,11 +904,6 @@ public abstract class BaseConfig { return new DaoSearchParamSynchronizer(); } - @Bean - public DeleteExpungeService deleteExpungeService() { - return new DeleteExpungeService(); - } - @Bean public ResourceTableFKProvider resourceTableFKProvider() { return new ResourceTableFKProvider(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IMdmLinkDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IMdmLinkDao.java index 969ea3926ca..468dbbec24e 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IMdmLinkDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IMdmLinkDao.java @@ -20,15 +20,16 @@ package ca.uhn.fhir.jpa.dao.data; * #L% */ -import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; import ca.uhn.fhir.jpa.entity.MdmLink; -import org.springframework.beans.factory.annotation.Value; +import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import java.util.Date; import java.util.List; @Repository @@ -70,4 +71,6 @@ public interface IMdmLinkDao extends JpaRepository { @Query("SELECT ml.myGoldenResourcePid as goldenPid, ml.mySourcePid as sourcePid FROM MdmLink ml WHERE ml.myGoldenResourcePid = :goldenPid and ml.myMatchResult = :matchResult") List expandPidsByGoldenResourcePidAndMatchResult(@Param("goldenPid") Long theSourcePid, @Param("matchResult") MdmMatchResultEnum theMdmMatchResultEnum); + @Query("SELECT ml.myId FROM MdmLink ml WHERE ml.myMdmSourceType = :resourceName AND ml.myCreated <= :highThreshold ORDER BY ml.myCreated DESC") + List findPidByResourceNameAndThreshold(@Param("resourceName") String theResourceName, @Param("highThreshold") Date theHighThreshold, Pageable thePageable); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeService.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeService.java deleted file mode 100644 index 21bcd2cddbb..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeService.java +++ /dev/null @@ -1,201 +0,0 @@ -package ca.uhn.fhir.jpa.dao.expunge; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2021 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.interceptor.api.HookParams; -import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; -import ca.uhn.fhir.interceptor.api.Pointcut; -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome; -import ca.uhn.fhir.jpa.dao.BaseHapiFhirResourceDao; -import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao; -import ca.uhn.fhir.jpa.dao.index.IdHelperService; -import ca.uhn.fhir.jpa.delete.job.DeleteExpungeProcessor; -import ca.uhn.fhir.jpa.model.entity.ResourceLink; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; -import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; -import ca.uhn.fhir.util.OperationOutcomeUtil; -import ca.uhn.fhir.util.StopWatch; -import org.hl7.fhir.instance.model.api.IBaseOperationOutcome; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Slice; -import org.springframework.stereotype.Service; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.support.TransactionTemplate; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.PersistenceContextType; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; - -@Service -/** - * DeleteExpunge is now performed using the {@link ca.uhn.fhir.jpa.delete.DeleteExpungeJobSubmitterImpl} Spring Batch job. - */ -@Deprecated -public class DeleteExpungeService { - private static final Logger ourLog = LoggerFactory.getLogger(DeleteExpungeService.class); - - @Autowired - protected PlatformTransactionManager myPlatformTransactionManager; - @PersistenceContext(type = PersistenceContextType.TRANSACTION) - private EntityManager myEntityManager; - @Autowired - private FhirContext myFhirContext; - @Autowired - private ResourceTableFKProvider myResourceTableFKProvider; - @Autowired - private IResourceLinkDao myResourceLinkDao; - @Autowired - private IInterceptorBroadcaster myInterceptorBroadcaster; - @Autowired - private DaoConfig myDaoConfig; - @Autowired - private IdHelperService myIdHelper; - - public DeleteMethodOutcome expungeByResourcePids(String theUrl, String theResourceName, Slice thePids, RequestDetails theRequest) { - StopWatch w = new StopWatch(); - if (thePids.isEmpty()) { - return new DeleteMethodOutcome(); - } - - HookParams params = new HookParams() - .add(RequestDetails.class, theRequest) - .addIfMatchesType(ServletRequestDetails.class, theRequest) - .add(String.class, theUrl); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PRE_DELETE_EXPUNGE, params); - - TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager); - txTemplate.executeWithoutResult(t -> validateOkToDeleteAndExpunge(thePids)); - - ourLog.info("Expunging all records linking to {} resources...", thePids.getNumber()); - AtomicLong expungedEntitiesCount = new AtomicLong(); - AtomicLong expungedResourcesCount = new AtomicLong(); - PartitionRunner partitionRunner = new PartitionRunner(DeleteExpungeProcessor.PROCESS_NAME, DeleteExpungeProcessor.THREAD_PREFIX, myDaoConfig.getExpungeBatchSize(), myDaoConfig.getExpungeThreadCount()); - partitionRunner.runInPartitionedThreads(thePids, pidChunk -> deleteInTransaction(theResourceName, pidChunk, expungedResourcesCount, expungedEntitiesCount, theRequest)); - ourLog.info("Expunged a total of {} records", expungedEntitiesCount); - - IBaseOperationOutcome oo; - if (expungedResourcesCount.get() == 0) { - oo = OperationOutcomeUtil.newInstance(myFhirContext); - String message = myFhirContext.getLocalizer().getMessageSanitized(BaseHapiFhirResourceDao.class, "unableToDeleteNotFound", theUrl); - String severity = "warning"; - String code = "not-found"; - OperationOutcomeUtil.addIssue(myFhirContext, oo, severity, message, null, code); - } else { - oo = OperationOutcomeUtil.newInstance(myFhirContext); - String message = myFhirContext.getLocalizer().getMessage(BaseHapiFhirResourceDao.class, "successfulDeletes", expungedResourcesCount.get(), w.getMillis()); - String severity = "information"; - String code = "informational"; - OperationOutcomeUtil.addIssue(myFhirContext, oo, severity, message, null, code); - } - - DeleteMethodOutcome retval = new DeleteMethodOutcome(); - retval.setExpungedResourcesCount(expungedResourcesCount.get()); - retval.setExpungedEntitiesCount(expungedEntitiesCount.get()); - retval.setOperationOutcome(oo); - return retval; - } - - public void validateOkToDeleteAndExpunge(Slice theAllTargetPids) { - if (!myDaoConfig.isEnforceReferentialIntegrityOnDelete()) { - ourLog.info("Referential integrity on delete disabled. Skipping referential integrity check."); - return; - } - - List conflictResourceLinks = Collections.synchronizedList(new ArrayList<>()); - PartitionRunner partitionRunner = new PartitionRunner(DeleteExpungeProcessor.PROCESS_NAME, DeleteExpungeProcessor.THREAD_PREFIX, myDaoConfig.getExpungeBatchSize(), myDaoConfig.getExpungeThreadCount()); - partitionRunner.runInPartitionedThreads(theAllTargetPids, someTargetPids -> findResourceLinksWithTargetPidIn(theAllTargetPids.getContent(), someTargetPids, conflictResourceLinks)); - - if (conflictResourceLinks.isEmpty()) { - return; - } - - ResourceLink firstConflict = conflictResourceLinks.get(0); - - //NB-GGG: We previously instantiated these ID values from firstConflict.getSourceResource().getIdDt(), but in a situation where we - //actually had to run delete conflict checks in multiple partitions, the executor service starts its own sessions on a per thread basis, and by the time - //we arrive here, those sessions are closed. So instead, we resolve them from PIDs, which are eagerly loaded. - String sourceResourceId = myIdHelper.resourceIdFromPidOrThrowException(firstConflict.getSourceResourcePid()).toVersionless().getValue(); - String targetResourceId = myIdHelper.resourceIdFromPidOrThrowException(firstConflict.getTargetResourcePid()).toVersionless().getValue(); - - throw new InvalidRequestException("DELETE with _expunge=true failed. Unable to delete " + - targetResourceId + " because " + sourceResourceId + " refers to it via the path " + firstConflict.getSourcePath()); - } - - public void findResourceLinksWithTargetPidIn(List theAllTargetPids, List theSomeTargetPids, List theConflictResourceLinks) { - // We only need to find one conflict, so if we found one already in an earlier partition run, we can skip the rest of the searches - if (theConflictResourceLinks.isEmpty()) { - List conflictResourceLinks = myResourceLinkDao.findWithTargetPidIn(theSomeTargetPids).stream() - // Filter out resource links for which we are planning to delete the source. - // theAllTargetPids contains a list of all the pids we are planning to delete. So we only want - // to consider a link to be a conflict if the source of that link is not in theAllTargetPids. - .filter(link -> !theAllTargetPids.contains(link.getSourceResourcePid())) - .collect(Collectors.toList()); - - // We do this in two steps to avoid lock contention on this synchronized list - theConflictResourceLinks.addAll(conflictResourceLinks); - } - } - - private void deleteInTransaction(String theResourceName, List thePidChunk, AtomicLong theExpungedResourcesCount, AtomicLong theExpungedEntitiesCount, RequestDetails theRequest) { - TransactionTemplate txTemplate = new TransactionTemplate(myPlatformTransactionManager); - txTemplate.executeWithoutResult(t -> deleteAllRecordsLinkingTo(theResourceName, thePidChunk, theExpungedResourcesCount, theExpungedEntitiesCount, theRequest)); - } - - private void deleteAllRecordsLinkingTo(String theResourceName, List thePids, AtomicLong theExpungedResourcesCount, AtomicLong theExpungedEntitiesCount, RequestDetails theRequest) { - HookParams params = new HookParams() - .add(String.class, theResourceName) - .add(List.class, thePids) - .add(AtomicLong.class, theExpungedEntitiesCount) - .add(RequestDetails.class, theRequest) - .addIfMatchesType(ServletRequestDetails.class, theRequest); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PRE_DELETE_EXPUNGE_PID_LIST, params); - - String pidListString = thePids.toString().replace("[", "(").replace("]", ")"); - List resourceForeignKeys = myResourceTableFKProvider.getResourceForeignKeys(); - - for (ResourceForeignKey resourceForeignKey : resourceForeignKeys) { - deleteRecordsByColumn(pidListString, resourceForeignKey, theExpungedEntitiesCount); - } - - // Lastly we need to delete records from the resource table all of these other tables link to: - ResourceForeignKey resourceTablePk = new ResourceForeignKey("HFJ_RESOURCE", "RES_ID"); - int entitiesDeleted = deleteRecordsByColumn(pidListString, resourceTablePk, theExpungedEntitiesCount); - theExpungedResourcesCount.addAndGet(entitiesDeleted); - } - - private int deleteRecordsByColumn(String thePidListString, ResourceForeignKey theResourceForeignKey, AtomicLong theExpungedEntitiesCount) { - int entitesDeleted = myEntityManager.createNativeQuery("DELETE FROM " + theResourceForeignKey.table + " WHERE " + theResourceForeignKey.key + " IN " + thePidListString).executeUpdate(); - ourLog.info("Expunged {} records from {}", entitesDeleted, theResourceForeignKey.table); - theExpungedEntitiesCount.addAndGet(entitesDeleted); - return entitesDeleted; - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmLinkExpandSvc.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmLinkExpandSvc.java index 4a28be40c5a..145b6be175f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmLinkExpandSvc.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/mdm/MdmLinkExpandSvc.java @@ -45,9 +45,12 @@ public class MdmLinkExpandSvc { @Autowired private IdHelperService myIdHelperService; + public MdmLinkExpandSvc() { + } + /** - * Given a source resource, perform MDM expansion and return all the resource IDs of all resources that are - * MDM-Matched to this resource. + * Given a source resource, perform MDM expansion and return all the resource IDs of all resources that are + * MDM-Matched to this resource. * * @param theResource The resource to MDM-Expand * @return A set of strings representing the FHIR IDs of the expanded resources. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java index 340be4372bc..756aab743e4 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java @@ -21,8 +21,9 @@ package ca.uhn.fhir.jpa.delete.job; */ import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.batch.job.MultiUrlProcessorJobConfig; +import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; +import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import org.springframework.batch.core.Job; @@ -45,7 +46,7 @@ import static ca.uhn.fhir.jpa.batch.BatchJobsConfig.DELETE_EXPUNGE_JOB_NAME; * Delete Expunge job. */ @Configuration -public class DeleteExpungeJobConfig extends MultiUrlProcessorJobConfig { +public class DeleteExpungeJobConfig { public static final String DELETE_EXPUNGE_URL_LIST_STEP_NAME = "delete-expunge-url-list-step"; @Autowired @@ -53,11 +54,23 @@ public class DeleteExpungeJobConfig extends MultiUrlProcessorJobConfig { @Autowired private JobBuilderFactory myJobBuilderFactory; + @Autowired + private MultiUrlJobParameterValidator myMultiUrlProcessorParameterValidator; + + @Autowired + private PidReaderCounterListener myPidCountRecorderListener; + + @Autowired + private ReverseCronologicalBatchResourcePidReader myReverseCronologicalBatchResourcePidReader; + + @Autowired + private SqlExecutorWriter mySqlExecutorWriter; + @Bean(name = DELETE_EXPUNGE_JOB_NAME) @Lazy - public Job deleteExpungeJob(MatchUrlService theMatchUrlService, DaoRegistry theDaoRegistry) { + public Job deleteExpungeJob() { return myJobBuilderFactory.get(DELETE_EXPUNGE_JOB_NAME) - .validator(multiUrlProcessorParameterValidator(theMatchUrlService, theDaoRegistry)) + .validator(myMultiUrlProcessorParameterValidator) .start(deleteExpungeUrlListStep()) .build(); } @@ -66,10 +79,10 @@ public class DeleteExpungeJobConfig extends MultiUrlProcessorJobConfig { public Step deleteExpungeUrlListStep() { return myStepBuilderFactory.get(DELETE_EXPUNGE_URL_LIST_STEP_NAME) ., List>chunk(1) - .reader(reverseCronologicalBatchResourcePidReader()) + .reader(myReverseCronologicalBatchResourcePidReader) .processor(deleteExpungeProcessor()) - .writer(sqlExecutorWriter()) - .listener(pidCountRecorderListener()) + .writer(mySqlExecutorWriter) + .listener(myPidCountRecorderListener) .listener(deleteExpungePromotionListener()) .build(); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexEverythingJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexEverythingJobConfig.java index ec66e2a3e0d..33fdfaf8ece 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexEverythingJobConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexEverythingJobConfig.java @@ -51,6 +51,8 @@ public class ReindexEverythingJobConfig { private JobBuilderFactory myJobBuilderFactory; @Autowired private ReindexWriter myReindexWriter; + @Autowired + private PidReaderCounterListener myPidCountRecorderListener; @Bean(name = REINDEX_EVERYTHING_JOB_NAME) @Lazy @@ -66,7 +68,7 @@ public class ReindexEverythingJobConfig { ., List>chunk(1) .reader(cronologicalBatchAllResourcePidReader()) .writer(myReindexWriter) - .listener(reindexEverythingPidCountRecorderListener()) + .listener(myPidCountRecorderListener) .listener(reindexEverythingPromotionListener()) .build(); } @@ -77,12 +79,6 @@ public class ReindexEverythingJobConfig { return new CronologicalBatchAllResourcePidReader(); } - @Bean - @StepScope - public PidReaderCounterListener reindexEverythingPidCountRecorderListener() { - return new PidReaderCounterListener(); - } - @Bean public ExecutionContextPromotionListener reindexEverythingPromotionListener() { ExecutionContextPromotionListener listener = new ExecutionContextPromotionListener(); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexJobConfig.java index ee62e9a0d93..c5465ec3533 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexJobConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/job/ReindexJobConfig.java @@ -21,15 +21,17 @@ package ca.uhn.fhir.jpa.reindex.job; */ import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.batch.job.MultiUrlProcessorJobConfig; +import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; +import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; +import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.core.listener.ExecutionContextPromotionListener; +import org.springframework.batch.item.ItemReader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -44,7 +46,7 @@ import static ca.uhn.fhir.jpa.batch.BatchJobsConfig.REINDEX_JOB_NAME; * Reindex job. */ @Configuration -public class ReindexJobConfig extends MultiUrlProcessorJobConfig { +public class ReindexJobConfig { public static final String REINDEX_URL_LIST_STEP_NAME = "reindex-url-list-step"; @Autowired @@ -54,11 +56,20 @@ public class ReindexJobConfig extends MultiUrlProcessorJobConfig { @Autowired private ReindexWriter myReindexWriter; + @Autowired + private MultiUrlJobParameterValidator myMultiUrlProcessorParameterValidator; + + @Autowired + private PidReaderCounterListener myPidCountRecorderListener; + + @Autowired + private ReverseCronologicalBatchResourcePidReader myReverseCronologicalBatchResourcePidReader; + @Bean(name = REINDEX_JOB_NAME) @Lazy - public Job reindexJob(MatchUrlService theMatchUrlService, DaoRegistry theDaoRegistry) { + public Job reindexJob() { return myJobBuilderFactory.get(REINDEX_JOB_NAME) - .validator(multiUrlProcessorParameterValidator(theMatchUrlService, theDaoRegistry)) + .validator(myMultiUrlProcessorParameterValidator) .start(reindexUrlListStep()) .build(); } @@ -67,9 +78,9 @@ public class ReindexJobConfig extends MultiUrlProcessorJobConfig { public Step reindexUrlListStep() { return myStepBuilderFactory.get(REINDEX_URL_LIST_STEP_NAME) ., List>chunk(1) - .reader(reverseCronologicalBatchResourcePidReader()) + .reader(myReverseCronologicalBatchResourcePidReader) .writer(myReindexWriter) - .listener(pidCountRecorderListener()) + .listener(myPidCountRecorderListener) .listener(reindexPromotionListener()) .build(); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java index ab74d2c3c0d..019e799e760 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportSvcImplR4Test.java @@ -331,7 +331,8 @@ public class BulkDataExportSvcImplR4Test extends BaseJpaR4Test { BatchJobsConfig.BULK_EXPORT_JOB_NAME, BatchJobsConfig.PATIENT_BULK_EXPORT_JOB_NAME, BatchJobsConfig.GROUP_BULK_EXPORT_JOB_NAME, - BatchJobsConfig.DELETE_EXPUNGE_JOB_NAME + BatchJobsConfig.DELETE_EXPUNGE_JOB_NAME, + BatchJobsConfig.MDM_CLEAR_JOB_NAME ); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java index 6c1b52518bc..61018ef82b1 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/delete/job/ReindexJobTest.java @@ -1,9 +1,9 @@ package ca.uhn.fhir.jpa.delete.job; import ca.uhn.fhir.jpa.batch.BatchJobsConfig; +import ca.uhn.fhir.jpa.batch.CommonBatchJobConfig; import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterUtil; -import ca.uhn.fhir.jpa.batch.job.MultiUrlProcessorJobConfig; import ca.uhn.fhir.jpa.batch.reader.CronologicalBatchAllResourcePidReader; import ca.uhn.fhir.jpa.dao.r4.BaseJpaR4Test; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; @@ -111,7 +111,7 @@ public class ReindexJobTest extends BaseJpaR4Test { private JobParameters buildEverythingJobParameters(Long theBatchSize) { Map map = new HashMap<>(); - map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), MultiUrlProcessorJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); + map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), CommonBatchJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_BATCH_SIZE, new JobParameter(theBatchSize.longValue())); JobParameters parameters = new JobParameters(map); return parameters; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java index 994ad38f17f..652ceff7947 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java @@ -15,6 +15,7 @@ import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.test.utilities.BatchJobHelper; import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; import org.hl7.fhir.r4.model.BooleanType; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.DecimalType; @@ -99,8 +100,7 @@ public class MultitenantBatchOperationR4Test extends BaseMultitenantResourceProv assertEquals("Patient", interceptor.resourceDefs.get(0).getName()); myInterceptorRegistry.unregisterInterceptor(interceptor); - DecimalType jobIdPrimitive = (DecimalType) response.getParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_RESPONSE_JOB_ID); - Long jobId = jobIdPrimitive.getValue().longValue(); + Long jobId = BatchHelperR4.jobIdFromParameters(response); assertEquals(1, myBatchJobHelper.getReadCount(jobId)); assertEquals(1, myBatchJobHelper.getWriteCount(jobId)); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java index 691ddc1319d..7b8a3bf32b8 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java @@ -55,6 +55,7 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.Bundle.BundleType; import org.hl7.fhir.r4.model.Bundle.HTTPVerb; @@ -817,8 +818,7 @@ public class SystemProviderR4Test extends BaseJpaR4Test { ourLog.info(ourCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(response)); myBatchJobHelper.awaitAllBulkJobCompletions(BatchJobsConfig.DELETE_EXPUNGE_JOB_NAME); - DecimalType jobIdPrimitive = (DecimalType) response.getParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_RESPONSE_JOB_ID); - Long jobId = jobIdPrimitive.getValue().longValue(); + Long jobId = BatchHelperR4.jobIdFromParameters(response); // validate diff --git a/hapi-fhir-jpaserver-batch/pom.xml b/hapi-fhir-jpaserver-batch/pom.xml index 660fdb5320a..7d7c41e39ce 100644 --- a/hapi-fhir-jpaserver-batch/pom.xml +++ b/hapi-fhir-jpaserver-batch/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-cql/pom.xml b/hapi-fhir-jpaserver-cql/pom.xml index 5e7a38e555b..37991c7f946 100644 --- a/hapi-fhir-jpaserver-cql/pom.xml +++ b/hapi-fhir-jpaserver-cql/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-mdm/pom.xml b/hapi-fhir-jpaserver-mdm/pom.xml index 8970e99afbf..ecb9bee6d63 100644 --- a/hapi-fhir-jpaserver-mdm/pom.xml +++ b/hapi-fhir-jpaserver-mdm/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java index 51981cdecaa..3005c2d5735 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmConsumerConfig.java @@ -22,6 +22,7 @@ package ca.uhn.fhir.jpa.mdm.config; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.batch.mdm.MdmBatchJobSubmitterFactoryImpl; import ca.uhn.fhir.jpa.dao.mdm.MdmLinkDeleteSvc; import ca.uhn.fhir.jpa.interceptor.MdmSearchExpandingInterceptor; import ca.uhn.fhir.jpa.mdm.broker.MdmMessageHandler; @@ -31,10 +32,8 @@ import ca.uhn.fhir.jpa.mdm.dao.MdmLinkFactory; import ca.uhn.fhir.jpa.mdm.interceptor.IMdmStorageInterceptor; import ca.uhn.fhir.jpa.mdm.interceptor.MdmStorageInterceptor; import ca.uhn.fhir.jpa.mdm.svc.GoldenResourceMergerSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmClearSvcImpl; import ca.uhn.fhir.jpa.mdm.svc.MdmControllerSvcImpl; import ca.uhn.fhir.jpa.mdm.svc.MdmEidUpdateService; -import ca.uhn.fhir.jpa.mdm.svc.MdmGoldenResourceDeletingSvc; import ca.uhn.fhir.jpa.mdm.svc.MdmLinkQuerySvcImpl; import ca.uhn.fhir.jpa.mdm.svc.MdmLinkSvcImpl; import ca.uhn.fhir.jpa.mdm.svc.MdmLinkUpdaterSvcImpl; @@ -52,8 +51,8 @@ import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmCandidateSearchCriteriaBuilderSvc; import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmCandidateSearchSvc; import ca.uhn.fhir.jpa.mdm.svc.candidate.MdmGoldenResourceFindingSvc; import ca.uhn.fhir.mdm.api.IGoldenResourceMergerSvc; +import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; -import ca.uhn.fhir.mdm.api.IMdmExpungeSvc; import ca.uhn.fhir.mdm.api.IMdmLinkQuerySvc; import ca.uhn.fhir.mdm.api.IMdmLinkSvc; import ca.uhn.fhir.mdm.api.IMdmLinkUpdaterSvc; @@ -183,8 +182,8 @@ public class MdmConsumerConfig { } @Bean - IMdmExpungeSvc mdmResetSvc(MdmLinkDaoSvc theMdmLinkDaoSvc, MdmGoldenResourceDeletingSvc theDeletingSvc, IMdmSettings theIMdmSettings) { - return new MdmClearSvcImpl(theMdmLinkDaoSvc, theDeletingSvc, theIMdmSettings); + IMdmBatchJobSubmitterFactory mdmBatchJobSubmitterFactory() { + return new MdmBatchJobSubmitterFactoryImpl(); } @Bean @@ -251,4 +250,5 @@ public class MdmConsumerConfig { IMdmControllerSvc mdmControllerSvc() { return new MdmControllerSvcImpl(); } + } diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmSubmitterConfig.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmSubmitterConfig.java index be9bb320ecd..e007b080081 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmSubmitterConfig.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmSubmitterConfig.java @@ -21,17 +21,15 @@ package ca.uhn.fhir.jpa.mdm.config; */ import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.mdm.api.IMdmChannelSubmitterSvc; -import ca.uhn.fhir.mdm.api.IMdmSettings; -import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; -import ca.uhn.fhir.mdm.rules.config.MdmRuleValidator; import ca.uhn.fhir.jpa.dao.mdm.MdmLinkDeleteSvc; import ca.uhn.fhir.jpa.mdm.interceptor.MdmSubmitterInterceptorLoader; import ca.uhn.fhir.jpa.mdm.svc.MdmChannelSubmitterSvcImpl; -import ca.uhn.fhir.jpa.mdm.svc.MdmGoldenResourceDeletingSvc; import ca.uhn.fhir.jpa.mdm.svc.MdmSearchParamSvc; import ca.uhn.fhir.jpa.mdm.svc.MdmSubmitSvcImpl; import ca.uhn.fhir.jpa.subscription.channel.api.IChannelFactory; +import ca.uhn.fhir.mdm.api.IMdmChannelSubmitterSvc; +import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; +import ca.uhn.fhir.mdm.rules.config.MdmRuleValidator; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -60,11 +58,6 @@ public class MdmSubmitterConfig { return new MdmLinkDeleteSvc(); } - @Bean - MdmGoldenResourceDeletingSvc mdmGoldenResourceDeletingSvc() { - return new MdmGoldenResourceDeletingSvc(); - } - @Bean @Lazy IMdmChannelSubmitterSvc mdmChannelSubmitterSvc(FhirContext theFhirContext, IChannelFactory theChannelFactory) { @@ -72,7 +65,7 @@ public class MdmSubmitterConfig { } @Bean - IMdmSubmitSvc mdmBatchService(IMdmSettings theMdmSetting) { + IMdmSubmitSvc mdmSubmitService() { return new MdmSubmitSvcImpl(); } } diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java index 45df193ddeb..8f72e867bc7 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/dao/MdmLinkDaoSvc.java @@ -40,13 +40,10 @@ import org.springframework.transaction.annotation.Transactional; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; public class MdmLinkDaoSvc { @@ -227,46 +224,6 @@ public class MdmLinkDaoSvc { return myMdmLinkDao.findAll(example); } - /** - * Delete all {@link MdmLink} entities, and return all resource PIDs from the source of the relationship. - * - * @return A list of Long representing the related Golden Resource Pids. - */ - @Transactional - public List deleteAllMdmLinksAndReturnGoldenResourcePids() { - List all = myMdmLinkDao.findAll(); - return deleteMdmLinksAndReturnGoldenResourcePids(all); - } - - private List deleteMdmLinksAndReturnGoldenResourcePids(List theLinks) { - Set goldenResources = theLinks.stream().map(MdmLink::getGoldenResourcePid).collect(Collectors.toSet()); - //TODO GGG this is probably invalid... we are essentially looking for GOLDEN -> GOLDEN links, which are either POSSIBLE_DUPLICATE - //and REDIRECT - goldenResources.addAll(theLinks.stream() - .filter(link -> link.getMatchResult().equals(MdmMatchResultEnum.REDIRECT) - || link.getMatchResult().equals(MdmMatchResultEnum.POSSIBLE_DUPLICATE)) - .map(MdmLink::getSourcePid).collect(Collectors.toSet())); - ourLog.info("Deleting {} MDM link records...", theLinks.size()); - myMdmLinkDao.deleteAll(theLinks); - ourLog.info("{} MDM link records deleted", theLinks.size()); - return new ArrayList<>(goldenResources); - } - - /** - * Given a valid {@link String}, delete all {@link MdmLink} entities for that type, and get the Pids - * for the Golden Resources which were the sources of the links. - * - * @param theSourceType the type of relationship you would like to delete. - * @return A list of longs representing the Pids of the Golden Resources resources used as the sources of the relationships that were deleted. - */ - public List deleteAllMdmLinksOfTypeAndReturnGoldenResourcePids(String theSourceType) { - MdmLink link = new MdmLink(); - link.setMdmSourceType(theSourceType); - Example exampleLink = Example.of(link); - List allOfType = myMdmLinkDao.findAll(exampleLink); - return deleteMdmLinksAndReturnGoldenResourcePids(allOfType); - } - /** * Persist an MDM link to the database. * diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmClearSvcImpl.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmClearSvcImpl.java deleted file mode 100644 index 44bfb9fd476..00000000000 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmClearSvcImpl.java +++ /dev/null @@ -1,84 +0,0 @@ -package ca.uhn.fhir.jpa.mdm.svc; - -/*- - * #%L - * HAPI FHIR JPA Server - Master Data Management - * %% - * Copyright (C) 2014 - 2021 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.mdm.api.IMdmExpungeSvc; -import ca.uhn.fhir.mdm.api.IMdmSettings; -import ca.uhn.fhir.mdm.log.Logs; -import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome; -import ca.uhn.fhir.jpa.mdm.dao.MdmLinkDaoSvc; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.rest.server.provider.ProviderConstants; -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; -import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.List; - -/** - * This class is responsible for clearing out existing MDM links, as well as deleting all Golden Resources related to those MDM Links. - */ -public class MdmClearSvcImpl implements IMdmExpungeSvc { - private static final Logger ourLog = Logs.getMdmTroubleshootingLog(); - - final MdmLinkDaoSvc myMdmLinkDaoSvc; - final MdmGoldenResourceDeletingSvc myMdmGoldenResourceDeletingSvcImpl; - final IMdmSettings myMdmSettings; - - @Autowired - public MdmClearSvcImpl(MdmLinkDaoSvc theMdmLinkDaoSvc, MdmGoldenResourceDeletingSvc theMdmGoldenResourceDeletingSvcImpl, IMdmSettings theIMdmSettings) { - myMdmLinkDaoSvc = theMdmLinkDaoSvc; - myMdmGoldenResourceDeletingSvcImpl = theMdmGoldenResourceDeletingSvcImpl; - myMdmSettings = theIMdmSettings; - } - - @Override - public long expungeAllMdmLinksOfSourceType(String theSourceResourceType, ServletRequestDetails theRequestDetails) { - throwExceptionIfInvalidSourceResourceType(theSourceResourceType); - ourLog.info("Clearing all MDM Links for resource type {}...", theSourceResourceType); - List goldenResourcePids = myMdmLinkDaoSvc.deleteAllMdmLinksOfTypeAndReturnGoldenResourcePids(theSourceResourceType); - DeleteMethodOutcome deleteOutcome = myMdmGoldenResourceDeletingSvcImpl.expungeGoldenResourcePids(goldenResourcePids, theSourceResourceType, theRequestDetails); - ourLog.info("MDM clear operation complete. Removed {} MDM links and {} Golden Resources.", goldenResourcePids.size(), deleteOutcome.getExpungedResourcesCount()); - return goldenResourcePids.size(); - } - - private void throwExceptionIfInvalidSourceResourceType(String theResourceType) { - if (!myMdmSettings.isSupportedMdmType(theResourceType)) { - throw new InvalidRequestException(ProviderConstants.MDM_CLEAR + " does not support resource type: " + theResourceType); - } - } - - @Override - public long expungeAllMdmLinks(ServletRequestDetails theRequestDetails) { - ourLog.info("Clearing all MDM Links..."); - long retVal = 0; - - for(String mdmType : myMdmSettings.getMdmRules().getMdmTypes()) { - List goldenResourcePids = myMdmLinkDaoSvc.deleteAllMdmLinksAndReturnGoldenResourcePids(); - DeleteMethodOutcome deleteOutcome = myMdmGoldenResourceDeletingSvcImpl.expungeGoldenResourcePids(goldenResourcePids, null, theRequestDetails); - ourLog.info("MDM clear operation on type {} complete. Removed {} MDM links and expunged {} Golden resources.", mdmType, goldenResourcePids.size(), deleteOutcome.getExpungedResourcesCount()); - retVal += goldenResourcePids.size(); - } - ourLog.info("MDM clear completed expunged with a total of {} golden resources cleared.", retVal); - return retVal; - } -} - diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java index 9c7a149b261..d6eaab73eaf 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java @@ -20,7 +20,9 @@ package ca.uhn.fhir.jpa.mdm.svc; * #L% */ +import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.mdm.api.IGoldenResourceMergerSvc; +import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; import ca.uhn.fhir.mdm.api.IMdmLinkQuerySvc; import ca.uhn.fhir.mdm.api.IMdmLinkUpdaterSvc; @@ -31,14 +33,20 @@ import ca.uhn.fhir.mdm.api.paging.MdmPageRequest; import ca.uhn.fhir.mdm.model.MdmTransactionContext; import ca.uhn.fhir.mdm.provider.MdmControllerHelper; import ca.uhn.fhir.mdm.provider.MdmControllerUtil; +import ca.uhn.fhir.rest.server.provider.MultiUrlProcessor; import ca.uhn.fhir.rest.server.provider.ProviderConstants; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import org.hl7.fhir.instance.model.api.IAnyResource; +import org.hl7.fhir.instance.model.api.IBaseParameters; import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.stereotype.Service; import javax.annotation.Nullable; +import java.math.BigDecimal; +import java.util.List; /** * This class acts as a layer between MdmProviders and MDM services to support a REST API that's not a FHIR Operation API. @@ -46,6 +54,8 @@ import javax.annotation.Nullable; @Service public class MdmControllerSvcImpl implements IMdmControllerSvc { + @Autowired + FhirContext myFhirContext; @Autowired MdmControllerHelper myMdmControllerHelper; @Autowired @@ -54,6 +64,11 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc { IMdmLinkQuerySvc myMdmLinkQuerySvc; @Autowired IMdmLinkUpdaterSvc myIMdmLinkUpdaterSvc; + @Autowired + IMdmBatchJobSubmitterFactory myMdmBatchJobSubmitterFactory; + + public MdmControllerSvcImpl() { + } @Override public IAnyResource mergeGoldenResources(String theFromGoldenResourceId, String theToGoldenResourceId, IAnyResource theManuallyMergedGoldenResource, MdmTransactionContext theMdmTransactionContext) { @@ -91,6 +106,12 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc { return myIMdmLinkUpdaterSvc.updateLink(goldenResource, source, matchResult, theMdmTransactionContext); } + @Override + public IBaseParameters submitMdmClearJob(List theUrls, IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails) { + MultiUrlProcessor multiUrlProcessor = new MultiUrlProcessor(myFhirContext, myMdmBatchJobSubmitterFactory.getClearJobSubmitter()); + return multiUrlProcessor.processUrls(theUrls, multiUrlProcessor.getBatchSize(theBatchSize), theRequestDetails); + } + @Override public void notDuplicateGoldenResource(String theGoldenResourceId, String theTargetGoldenResourceId, MdmTransactionContext theMdmTransactionContext) { IAnyResource goldenResource = myMdmControllerHelper.getLatestGoldenResourceFromIdOrThrowException(ProviderConstants.MDM_UPDATE_LINK_GOLDEN_RESOURCE_ID, theGoldenResourceId); diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmGoldenResourceDeletingSvc.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmGoldenResourceDeletingSvc.java deleted file mode 100644 index 9b53ce9bce6..00000000000 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmGoldenResourceDeletingSvc.java +++ /dev/null @@ -1,57 +0,0 @@ -package ca.uhn.fhir.jpa.mdm.svc; - -/*- - * #%L - * HAPI FHIR JPA Server - Master Data Management - * %% - * Copyright (C) 2014 - 2021 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.mdm.log.Logs; -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome; -import ca.uhn.fhir.jpa.dao.expunge.DeleteExpungeService; -import ca.uhn.fhir.jpa.dao.expunge.ExpungeService; -import ca.uhn.fhir.rest.server.provider.ProviderConstants; -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; -import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.SliceImpl; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class MdmGoldenResourceDeletingSvc { - - private static final Logger ourLog = Logs.getMdmTroubleshootingLog(); - - /** - * This is here for the case of possible infinite loops. Technically batch conflict deletion should handle this, but this is an escape hatch. - */ - private static final int MAXIMUM_DELETE_ATTEMPTS = 100000; - - @Autowired - private DaoRegistry myDaoRegistry; - @Autowired - private ExpungeService myExpungeService; - @Autowired - DeleteExpungeService myDeleteExpungeService; - - public DeleteMethodOutcome expungeGoldenResourcePids(List theGoldenResourcePids, String theResourceType, ServletRequestDetails theRequestDetails) { - return myDeleteExpungeService.expungeByResourcePids(ProviderConstants.MDM_CLEAR, theResourceType, new SliceImpl<>(theGoldenResourcePids), theRequestDetails); - } -} diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmSubmitSvcImpl.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmSubmitSvcImpl.java index 075b6088780..83a0d542b3c 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmSubmitSvcImpl.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmSubmitSvcImpl.java @@ -20,10 +20,6 @@ package ca.uhn.fhir.jpa.mdm.svc; * #L% */ -import ca.uhn.fhir.mdm.api.IMdmChannelSubmitterSvc; -import ca.uhn.fhir.mdm.api.IMdmSettings; -import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; -import ca.uhn.fhir.mdm.log.Logs; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; @@ -31,6 +27,10 @@ import ca.uhn.fhir.jpa.dao.IResultIterator; import ca.uhn.fhir.jpa.dao.ISearchBuilder; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.mdm.api.IMdmChannelSubmitterSvc; +import ca.uhn.fhir.mdm.api.IMdmSettings; +import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; +import ca.uhn.fhir.mdm.log.Logs; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; @@ -69,6 +69,9 @@ public class MdmSubmitSvcImpl implements IMdmSubmitSvc { private int myBufferSize = DEFAULT_BUFFER_SIZE; + public MdmSubmitSvcImpl() { + } + @Override @Transactional public long submitAllSourceTypesToMdm(@Nullable String theCriteria) { diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/config/BaseTestMdmConfig.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/config/BaseTestMdmConfig.java index 0af4ba1aba7..51577a8d2d7 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/config/BaseTestMdmConfig.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/config/BaseTestMdmConfig.java @@ -1,13 +1,13 @@ package ca.uhn.fhir.jpa.mdm.config; +import ca.uhn.fhir.jpa.mdm.helper.MdmLinkHelper; import ca.uhn.fhir.mdm.api.IMdmSettings; import ca.uhn.fhir.mdm.rules.config.MdmRuleValidator; import ca.uhn.fhir.mdm.rules.config.MdmSettings; -import ca.uhn.fhir.jpa.mdm.helper.MdmLinkHelper; -import ca.uhn.fhir.rest.server.FifoMemoryPagingProvider; -import ca.uhn.fhir.rest.server.IPagingProvider; +import ca.uhn.fhir.test.utilities.BatchJobHelper; import com.google.common.base.Charsets; import org.apache.commons.io.IOUtils; +import org.springframework.batch.core.explore.JobExplorer; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -41,4 +41,9 @@ public abstract class BaseTestMdmConfig { MdmLinkHelper mdmLinkHelper() { return new MdmLinkHelper(); } + + @Bean + BatchJobHelper batchJobHelper(JobExplorer theJobExplorer) { + return new BatchJobHelper(theJobExplorer); + } } diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseLinkR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseLinkR4Test.java index d44a313340d..9f1092a941a 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseLinkR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseLinkR4Test.java @@ -1,9 +1,9 @@ package ca.uhn.fhir.jpa.mdm.provider; -import ca.uhn.fhir.mdm.api.MdmLinkSourceEnum; -import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.entity.MdmLink; +import ca.uhn.fhir.mdm.api.MdmLinkSourceEnum; +import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.StringType; @@ -51,6 +51,8 @@ public abstract class BaseLinkR4Test extends BaseProviderR4Test { saveLink(myLink); assertEquals(MdmLinkSourceEnum.AUTO, myLink.getLinkSource()); myDaoConfig.setExpungeEnabled(true); + myDaoConfig.setAllowMultipleDelete(true); + myDaoConfig.setDeleteExpungeEnabled(true); } @AfterEach diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java index 5cbec2e45af..09fc013195c 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java @@ -1,23 +1,29 @@ package ca.uhn.fhir.jpa.mdm.provider; import ca.uhn.fhir.jpa.mdm.BaseMdmR4Test; +import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; -import ca.uhn.fhir.mdm.api.IMdmExpungeSvc; import ca.uhn.fhir.mdm.api.IMdmMatchFinderSvc; import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; import ca.uhn.fhir.mdm.provider.MdmProviderDstu3Plus; import ca.uhn.fhir.mdm.rules.config.MdmSettings; -import ca.uhn.fhir.rest.server.IPagingProvider; -import ca.uhn.fhir.rest.server.IRestfulServerDefaults; +import ca.uhn.fhir.test.utilities.BatchJobHelper; import com.google.common.base.Charsets; import org.apache.commons.io.IOUtils; +import org.hl7.fhir.instance.model.api.IPrimitiveType; +import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.StringType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.Resource; +import javax.annotation.Nonnull; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public abstract class BaseProviderR4Test extends BaseMdmR4Test { MdmProviderDstu3Plus myMdmProvider; @@ -26,11 +32,13 @@ public abstract class BaseProviderR4Test extends BaseMdmR4Test { @Autowired private IMdmControllerSvc myMdmControllerSvc; @Autowired - private IMdmExpungeSvc myMdmExpungeSvc; + private IMdmClearJobSubmitter myMdmClearJobSubmitter; @Autowired private IMdmSubmitSvc myMdmSubmitSvc; @Autowired private MdmSettings myMdmSettings; + @Autowired + BatchJobHelper myBatchJobHelper; private String defaultScript; @@ -44,13 +52,31 @@ public abstract class BaseProviderR4Test extends BaseMdmR4Test { @BeforeEach public void before() { - myMdmProvider = new MdmProviderDstu3Plus(myFhirContext, myMdmControllerSvc, myMdmMatchFinderSvc, myMdmExpungeSvc, myMdmSubmitSvc); + myMdmProvider = new MdmProviderDstu3Plus(myFhirContext, myMdmControllerSvc, myMdmMatchFinderSvc, myMdmSubmitSvc, myMdmSettings); defaultScript = myMdmSettings.getScriptText(); } + @AfterEach public void after() throws IOException { super.after(); myMdmSettings.setScriptText(defaultScript); myMdmResourceMatcherSvc.init();// This bugger creates new objects from the beans and then ignores them. } + + protected void clearMdmLinks() { + Parameters result = (Parameters) myMdmProvider.clearMdmLinks(null, null, myRequestDetails); + myBatchJobHelper.awaitJobExecution(BatchHelperR4.jobIdFromParameters(result)); + } + + protected void clearMdmLinks(String theResourceName) { + Parameters result = (Parameters) myMdmProvider.clearMdmLinks(getResourceNames(theResourceName), null, myRequestDetails); + myBatchJobHelper.awaitJobExecution(BatchHelperR4.jobIdFromParameters(result)); + } + + @Nonnull + protected List> getResourceNames(String theResourceName) { + List> resourceNames = new ArrayList<>(); + resourceNames.add(new StringType(theResourceName)); + return resourceNames; + } } diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java index ca168bd378b..db593713c0c 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderBatchR4Test.java @@ -68,7 +68,7 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { @Test public void testBatchRunOnAllMedications() throws InterruptedException { StringType criteria = null; - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchOnAllSourceResources(new StringType("Medication"), criteria, null)); assertLinkCount(1); @@ -77,32 +77,33 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { @Test public void testBatchRunOnAllPractitioners() throws InterruptedException { StringType criteria = null; - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPractitionerType(criteria, null)); assertLinkCount(1); } @Test public void testBatchRunOnSpecificPractitioner() throws InterruptedException { - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPractitionerInstance(myPractitioner.getIdElement(), null)); assertLinkCount(1); } @Test public void testBatchRunOnNonExistentSpecificPractitioner() { - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); try { myMdmProvider.mdmBatchPractitionerInstance(new IdType("Practitioner/999"), null); fail(); - } catch (ResourceNotFoundException e){} + } catch (ResourceNotFoundException e) { + } } @Test public void testBatchRunOnAllPatients() throws InterruptedException { assertLinkCount(3); StringType criteria = null; - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPatientType(criteria, null)); assertLinkCount(1); } @@ -110,7 +111,7 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { @Test public void testBatchRunOnSpecificPatient() throws InterruptedException { assertLinkCount(3); - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); afterMdmLatch.runWithExpectedCount(1, () -> myMdmProvider.mdmBatchPatientInstance(myPatient.getIdElement(), null)); assertLinkCount(1); } @@ -118,18 +119,19 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { @Test public void testBatchRunOnNonExistentSpecificPatient() { assertLinkCount(3); - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); try { myMdmProvider.mdmBatchPatientInstance(new IdType("Patient/999"), null); fail(); - } catch (ResourceNotFoundException e){} + } catch (ResourceNotFoundException e) { + } } @Test public void testBatchRunOnAllTypes() throws InterruptedException { assertLinkCount(3); StringType criteria = new StringType(""); - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); afterMdmLatch.runWithExpectedCount(3, () -> { myMdmProvider.mdmBatchOnAllSourceResources(null, criteria, null); }); @@ -140,12 +142,12 @@ public class MdmProviderBatchR4Test extends BaseLinkR4Test { public void testBatchRunOnAllTypesWithInvalidCriteria() { assertLinkCount(3); StringType criteria = new StringType("death-date=2020-06-01"); - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); try { myMdmProvider.mdmBatchPractitionerType(criteria, null); fail(); - } catch(InvalidRequestException e) { + } catch (InvalidRequestException e) { assertThat(e.getMessage(), is(equalTo("Failed to parse match URL[death-date=2020-06-01] - Resource type Practitioner does not have a parameter with name: death-date"))); } } diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderClearLinkR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderClearLinkR4Test.java index e399b867e62..2ba25cf2952 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderClearLinkR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/MdmProviderClearLinkR4Test.java @@ -10,7 +10,6 @@ import ca.uhn.fhir.rest.param.TokenParam; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; import org.hl7.fhir.instance.model.api.IAnyResource; -import org.hl7.fhir.instance.model.api.IBaseParameters; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Practitioner; import org.hl7.fhir.r4.model.StringType; @@ -30,7 +29,6 @@ import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.fail; public class MdmProviderClearLinkR4Test extends BaseLinkR4Test { - protected Practitioner myPractitioner; protected StringType myPractitionerId; protected IAnyResource myPractitionerGoldenResource; @@ -48,7 +46,7 @@ public class MdmProviderClearLinkR4Test extends BaseLinkR4Test { @Test public void testClearAllLinks() { assertLinkCount(2); - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); assertNoLinksExist(); } @@ -70,12 +68,13 @@ public class MdmProviderClearLinkR4Test extends BaseLinkR4Test { assertLinkCount(2); Patient read = myPatientDao.read(new IdDt(mySourcePatientId.getValueAsString()).toVersionless()); assertThat(read, is(notNullValue())); - myMdmProvider.clearMdmLinks(new StringType("Patient"), myRequestDetails); + clearMdmLinks("Patient"); assertNoPatientLinksExist(); try { myPatientDao.read(new IdDt(mySourcePatientId.getValueAsString()).toVersionless()); fail(); - } catch (ResourceNotFoundException e) {} + } catch (ResourceNotFoundException e) { + } } @Test @@ -87,7 +86,7 @@ public class MdmProviderClearLinkR4Test extends BaseLinkR4Test { Patient patientAndUpdateLinks = createPatientAndUpdateLinks(buildJanePatient()); IAnyResource goldenResource = getGoldenResourceFromTargetResource(patientAndUpdateLinks); assertThat(goldenResource, is(notNullValue())); - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); assertNoPatientLinksExist(); goldenResource = getGoldenResourceFromTargetResource(patientAndUpdateLinks); assertThat(goldenResource, is(nullValue())); @@ -104,7 +103,7 @@ public class MdmProviderClearLinkR4Test extends BaseLinkR4Test { linkGoldenResources(goldenResourceFromTarget, goldenResourceFromTarget2); //SUT - myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); assertNoPatientLinksExist(); IBundleProvider search = myPatientDao.search(buildGoldenResourceParameterMap()); @@ -135,7 +134,7 @@ public class MdmProviderClearLinkR4Test extends BaseLinkR4Test { linkGoldenResources(goldenResourceFromTarget2, goldenResourceFromTarget); //SUT - IBaseParameters parameters = myMdmProvider.clearMdmLinks(null, myRequestDetails); + clearMdmLinks(); printLinks(); @@ -157,18 +156,19 @@ public class MdmProviderClearLinkR4Test extends BaseLinkR4Test { assertLinkCount(2); Practitioner read = myPractitionerDao.read(new IdDt(myPractitionerGoldenResourceId.getValueAsString()).toVersionless()); assertThat(read, is(notNullValue())); - myMdmProvider.clearMdmLinks(new StringType("Practitioner"), myRequestDetails); + clearMdmLinks("Practitioner"); assertNoPractitionerLinksExist(); try { myPractitionerDao.read(new IdDt(myPractitionerGoldenResourceId.getValueAsString()).toVersionless()); fail(); - } catch (ResourceNotFoundException e) {} + } catch (ResourceNotFoundException e) { + } } @Test public void testClearInvalidTargetType() { try { - myMdmProvider.clearMdmLinks(new StringType("Observation"), myRequestDetails); + myMdmProvider.clearMdmLinks(getResourceNames("Observation"), null, myRequestDetails); fail(); } catch (InvalidRequestException e) { assertThat(e.getMessage(), is(equalTo("$mdm-clear does not support resource type: Observation"))); diff --git a/hapi-fhir-jpaserver-migrate/pom.xml b/hapi-fhir-jpaserver-migrate/pom.xml index 549dda66e65..583bb43b316 100644 --- a/hapi-fhir-jpaserver-migrate/pom.xml +++ b/hapi-fhir-jpaserver-migrate/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-model/pom.xml b/hapi-fhir-jpaserver-model/pom.xml index 2329c0873f6..4c1f86a37fc 100644 --- a/hapi-fhir-jpaserver-model/pom.xml +++ b/hapi-fhir-jpaserver-model/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-searchparam/pom.xml b/hapi-fhir-jpaserver-searchparam/pom.xml index 1e10aec0e32..85e78caf1ba 100755 --- a/hapi-fhir-jpaserver-searchparam/pom.xml +++ b/hapi-fhir-jpaserver-searchparam/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-subscription/pom.xml b/hapi-fhir-jpaserver-subscription/pom.xml index 97181675c03..e3609e6f624 100644 --- a/hapi-fhir-jpaserver-subscription/pom.xml +++ b/hapi-fhir-jpaserver-subscription/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-utilities/pom.xml b/hapi-fhir-jpaserver-test-utilities/pom.xml index c1f8822b95b..c0fa455be9b 100644 --- a/hapi-fhir-jpaserver-test-utilities/pom.xml +++ b/hapi-fhir-jpaserver-test-utilities/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml index 51f3502fe96..9ef24cb24a2 100644 --- a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml +++ b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-server-mdm/pom.xml b/hapi-fhir-server-mdm/pom.xml index fcfdafec802..8a7136ac721 100644 --- a/hapi-fhir-server-mdm/pom.xml +++ b/hapi-fhir-server-mdm/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java new file mode 100644 index 00000000000..6291d89d99a --- /dev/null +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java @@ -0,0 +1,5 @@ +package ca.uhn.fhir.mdm.api; + +public interface IMdmBatchJobSubmitterFactory { + IMdmClearJobSubmitter getClearJobSubmitter(); +} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmExpungeSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmClearJobSubmitter.java similarity index 51% rename from hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmExpungeSvc.java rename to hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmClearJobSubmitter.java index be7766b3244..33f6ad83e8e 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmExpungeSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmClearJobSubmitter.java @@ -20,24 +20,10 @@ package ca.uhn.fhir.mdm.api; * #L% */ -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.rest.api.server.storage.IMultiUrlJobSubmitter; -public interface IMdmExpungeSvc { - - /** - * Given a resource type, delete the underlying MDM links, and their related golden resource objects. - * - * @param theSourceResourceType The type of resources - * @param theRequestDetails - * @return the count of deleted MDM links - */ - long expungeAllMdmLinksOfSourceType(String theSourceResourceType, ServletRequestDetails theRequestDetails); - - /** - * Delete all MDM links, and their related golden resource objects. - * - * @return the count of deleted MDM links - * @param theRequestDetails - */ - long expungeAllMdmLinks(ServletRequestDetails theRequestDetails); +/** + * Tag interface for Spring autowiring + */ +public interface IMdmClearJobSubmitter extends IMultiUrlJobSubmitter { } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java index 67a3dc64a53..fd2dd82e65d 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmControllerSvc.java @@ -22,10 +22,15 @@ package ca.uhn.fhir.mdm.api; import ca.uhn.fhir.mdm.api.paging.MdmPageRequest; import ca.uhn.fhir.mdm.model.MdmTransactionContext; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import org.hl7.fhir.instance.model.api.IAnyResource; +import org.hl7.fhir.instance.model.api.IBaseParameters; +import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.springframework.data.domain.Page; import javax.annotation.Nullable; +import java.math.BigDecimal; +import java.util.List; public interface IMdmControllerSvc { @@ -38,4 +43,6 @@ public interface IMdmControllerSvc { IAnyResource mergeGoldenResources(String theFromGoldenResourceId, String theToGoldenResourceId, IAnyResource theManuallyMergedGoldenResource, MdmTransactionContext theMdmTransactionContext); IAnyResource updateLink(String theGoldenResourceId, String theSourceResourceId, String theMatchResult, MdmTransactionContext theMdmTransactionContext); + + IBaseParameters submitMdmClearJob(List theUrls, IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails); } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java index 28aa52c69b5..b65427eedee 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderDstu3Plus.java @@ -22,8 +22,8 @@ package ca.uhn.fhir.mdm.provider; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; -import ca.uhn.fhir.mdm.api.IMdmExpungeSvc; import ca.uhn.fhir.mdm.api.IMdmMatchFinderSvc; +import ca.uhn.fhir.mdm.api.IMdmSettings; import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; import ca.uhn.fhir.mdm.api.MatchedTarget; import ca.uhn.fhir.mdm.api.MdmConstants; @@ -31,7 +31,6 @@ import ca.uhn.fhir.mdm.api.MdmLinkJson; import ca.uhn.fhir.mdm.api.paging.MdmPageRequest; import ca.uhn.fhir.mdm.model.MdmTransactionContext; import ca.uhn.fhir.model.api.annotation.Description; -import ca.uhn.fhir.model.primitive.IntegerDt; import ca.uhn.fhir.rest.annotation.IdParam; import ca.uhn.fhir.rest.annotation.Operation; import ca.uhn.fhir.rest.annotation.OperationParam; @@ -58,10 +57,12 @@ import org.springframework.data.domain.Page; import javax.annotation.Nonnull; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import static ca.uhn.fhir.rest.api.Constants.PARAM_OFFSET; import static org.slf4j.LoggerFactory.getLogger; @@ -69,11 +70,10 @@ import static org.slf4j.LoggerFactory.getLogger; public class MdmProviderDstu3Plus extends BaseMdmProvider { private static final Logger ourLog = getLogger(MdmProviderDstu3Plus.class); - private final IMdmControllerSvc myMdmControllerSvc; private final IMdmMatchFinderSvc myMdmMatchFinderSvc; - private final IMdmExpungeSvc myMdmExpungeSvc; private final IMdmSubmitSvc myMdmSubmitSvc; + private final IMdmSettings myMdmSettings; public static final int DEFAULT_PAGE_SIZE = 20; public static final int MAX_PAGE_SIZE = 100; @@ -84,12 +84,12 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { * Note that this is not a spring bean. Any necessary injections should * happen in the constructor */ - public MdmProviderDstu3Plus(FhirContext theFhirContext, IMdmControllerSvc theMdmControllerSvc, IMdmMatchFinderSvc theMdmMatchFinderSvc, IMdmExpungeSvc theMdmExpungeSvc, IMdmSubmitSvc theMdmSubmitSvc) { + public MdmProviderDstu3Plus(FhirContext theFhirContext, IMdmControllerSvc theMdmControllerSvc, IMdmMatchFinderSvc theMdmMatchFinderSvc, IMdmSubmitSvc theMdmSubmitSvc, IMdmSettings theIMdmSettings) { super(theFhirContext); myMdmControllerSvc = theMdmControllerSvc; myMdmMatchFinderSvc = theMdmMatchFinderSvc; - myMdmExpungeSvc = theMdmExpungeSvc; myMdmSubmitSvc = theMdmSubmitSvc; + myMdmSettings = theIMdmSettings; } @Operation(name = ProviderConstants.EMPI_MATCH, typeName = "Patient") @@ -180,21 +180,33 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { ); } - @Operation(name = ProviderConstants.MDM_CLEAR, returnParameters = { - @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT, typeName = "decimal") + @Operation(name = ProviderConstants.OPERATION_MDM_CLEAR, returnParameters = { + @OperationParam(name = ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, typeName = "decimal") }) - public IBaseParameters clearMdmLinks(@OperationParam(name = ProviderConstants.MDM_CLEAR_SOURCE_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType theSourceType, + public IBaseParameters clearMdmLinks(@OperationParam(name = ProviderConstants.OPERATION_MDM_CLEAR_RESOURCE_NAME, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theResourceNames, + @OperationParam(name = ProviderConstants.OPERATION_MDM_CLEAR_BATCH_SIZE, typeName = "decimal", min = 0, max = 1) IPrimitiveType theBatchSize, ServletRequestDetails theRequestDetails) { - long resetCount; - if (theSourceType == null || StringUtils.isBlank(theSourceType.getValue())) { - resetCount = myMdmExpungeSvc.expungeAllMdmLinks(theRequestDetails); + + List resourceNames = new ArrayList<>(); + + + if (theResourceNames != null) { + resourceNames.addAll(theResourceNames.stream().map(IPrimitiveType::getValue).collect(Collectors.toList())); + validateResourceNames(resourceNames); } else { - resetCount = myMdmExpungeSvc.expungeAllMdmLinksOfSourceType(theSourceType.getValueAsString(), theRequestDetails); + resourceNames.addAll(myMdmSettings.getMdmRules().getMdmTypes()); } - IBaseParameters retval = ParametersUtil.newInstance(myFhirContext); - ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_MDM_CLEAR_OUT_PARAM_DELETED_COUNT, resetCount); - return retval; + List urls = resourceNames.stream().map(s -> s + "?").collect(Collectors.toList()); + return myMdmControllerSvc.submitMdmClearJob(urls, theBatchSize, theRequestDetails); + } + + private void validateResourceNames(List theResourceNames) { + for (String resourceName : theResourceNames) { + if (!myMdmSettings.isSupportedMdmType(resourceName)) { + throw new InvalidRequestException(ProviderConstants.OPERATION_MDM_CLEAR + " does not support resource type: " + resourceName); + } + } } @Operation(name = ProviderConstants.MDM_QUERY_LINKS, idempotent = true) @@ -202,9 +214,9 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_RESOURCE_ID, min = 0, max = 1, typeName = "string") IPrimitiveType theResourceId, @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_MATCH_RESULT, min = 0, max = 1, typeName = "string") IPrimitiveType theMatchResult, @OperationParam(name = ProviderConstants.MDM_QUERY_LINKS_LINK_SOURCE, min = 0, max = 1, typeName = "string") - IPrimitiveType theLinkSource, + IPrimitiveType theLinkSource, - @Description(formalDefinition="Results from this method are returned across multiple pages. This parameter controls the offset when fetching a page.") + @Description(formalDefinition = "Results from this method are returned across multiple pages. This parameter controls the offset when fetching a page.") @OperationParam(name = PARAM_OFFSET, min = 0, max = 1, typeName = "integer") IPrimitiveType theOffset, @Description(formalDefinition = "Results from this method are returned across multiple pages. This parameter controls the size of those pages.") @@ -233,7 +245,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { MdmPageRequest mdmPageRequest = new MdmPageRequest(theOffset, theCount, DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE); - Page possibleDuplicates = myMdmControllerSvc.getDuplicateGoldenResources(createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.DUPLICATE_GOLDEN_RESOURCES, (String) null), mdmPageRequest); + Page possibleDuplicates = myMdmControllerSvc.getDuplicateGoldenResources(createMdmContext(theRequestDetails, MdmTransactionContext.OperationType.DUPLICATE_GOLDEN_RESOURCES, null), mdmPageRequest); return parametersFromMdmLinks(possibleDuplicates, false, theRequestDetails, mdmPageRequest); } @@ -255,7 +267,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { } @Operation(name = ProviderConstants.OPERATION_MDM_SUBMIT, idempotent = false, returnParameters = { - @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT, typeName = "integer") + @OperationParam(name = ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, typeName = "integer") }) public IBaseParameters mdmBatchOnAllSourceResources( @OperationParam(name = ProviderConstants.MDM_BATCH_RUN_RESOURCE_TYPE, min = 0, max = 1, typeName = "string") IPrimitiveType theResourceType, @@ -278,7 +290,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { @Operation(name = ProviderConstants.OPERATION_MDM_SUBMIT, idempotent = false, typeName = "Patient", returnParameters = { - @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT, typeName = "integer") + @OperationParam(name = ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, typeName = "integer") }) public IBaseParameters mdmBatchPatientInstance( @IdParam IIdType theIdParam, @@ -288,7 +300,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { } @Operation(name = ProviderConstants.OPERATION_MDM_SUBMIT, idempotent = false, typeName = "Patient", returnParameters = { - @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT, typeName = "integer") + @OperationParam(name = ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, typeName = "integer") }) public IBaseParameters mdmBatchPatientType( @OperationParam(name = ProviderConstants.MDM_BATCH_RUN_CRITERIA, typeName = "string") IPrimitiveType theCriteria, @@ -299,7 +311,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { } @Operation(name = ProviderConstants.OPERATION_MDM_SUBMIT, idempotent = false, typeName = "Practitioner", returnParameters = { - @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT, typeName = "integer") + @OperationParam(name = ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, typeName = "integer") }) public IBaseParameters mdmBatchPractitionerInstance( @IdParam IIdType theIdParam, @@ -309,7 +321,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { } @Operation(name = ProviderConstants.OPERATION_MDM_SUBMIT, idempotent = false, typeName = "Practitioner", returnParameters = { - @OperationParam(name = ProviderConstants.OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT, typeName = "integer") + @OperationParam(name = ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, typeName = "integer") }) public IBaseParameters mdmBatchPractitionerType( @OperationParam(name = ProviderConstants.MDM_BATCH_RUN_CRITERIA, typeName = "string") IPrimitiveType theCriteria, @@ -324,7 +336,7 @@ public class MdmProviderDstu3Plus extends BaseMdmProvider { */ public IBaseParameters buildMdmOutParametersWithCount(long theCount) { IBaseParameters retval = ParametersUtil.newInstance(myFhirContext); - ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT, theCount); + ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, theCount); return retval; } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java index e63e6b17d50..4765f045ea6 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/provider/MdmProviderLoader.java @@ -23,12 +23,9 @@ package ca.uhn.fhir.mdm.provider; import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; -import ca.uhn.fhir.mdm.api.IMdmExpungeSvc; import ca.uhn.fhir.mdm.api.IMdmMatchFinderSvc; +import ca.uhn.fhir.mdm.api.IMdmSettings; import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; -import ca.uhn.fhir.rest.server.IPagingProvider; -import ca.uhn.fhir.rest.server.IRestfulServerDefaults; -import ca.uhn.fhir.rest.server.RestfulServer; import ca.uhn.fhir.rest.server.provider.ResourceProviderFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -46,9 +43,9 @@ public class MdmProviderLoader { @Autowired private IMdmControllerSvc myMdmControllerSvc; @Autowired - private IMdmExpungeSvc myMdmExpungeSvc; - @Autowired private IMdmSubmitSvc myMdmSubmitSvc; + @Autowired + private IMdmSettings myMdmSettings; private BaseMdmProvider myMdmProvider; @@ -57,7 +54,7 @@ public class MdmProviderLoader { case DSTU3: case R4: myResourceProviderFactory.addSupplier(() -> { - myMdmProvider = new MdmProviderDstu3Plus(myFhirContext, myMdmControllerSvc, myMdmMatchFinderSvc, myMdmExpungeSvc, myMdmSubmitSvc); + myMdmProvider = new MdmProviderDstu3Plus(myFhirContext, myMdmControllerSvc, myMdmMatchFinderSvc, myMdmSubmitSvc, myMdmSettings); return myMdmProvider; }); break; diff --git a/hapi-fhir-server-openapi/pom.xml b/hapi-fhir-server-openapi/pom.xml index f2ad8061c43..ac3f9f6c8cc 100644 --- a/hapi-fhir-server-openapi/pom.xml +++ b/hapi-fhir-server-openapi/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server/pom.xml b/hapi-fhir-server/pom.xml index a559df05560..da732fe7ec6 100644 --- a/hapi-fhir-server/pom.xml +++ b/hapi-fhir-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/DeleteExpungeProvider.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/DeleteExpungeProvider.java index ee2feaee91d..faf7b7ef1b7 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/DeleteExpungeProvider.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/DeleteExpungeProvider.java @@ -32,9 +32,11 @@ import java.math.BigDecimal; import java.util.List; import java.util.stream.Collectors; -public class DeleteExpungeProvider extends BaseMultiUrlProcessor { +public class DeleteExpungeProvider { + private final MultiUrlProcessor myMultiUrlProcessor; + public DeleteExpungeProvider(FhirContext theFhirContext, IDeleteExpungeJobSubmitter theDeleteExpungeJobSubmitter) { - super(theFhirContext, theDeleteExpungeJobSubmitter); + myMultiUrlProcessor = new MultiUrlProcessor(theFhirContext, theDeleteExpungeJobSubmitter); } @Operation(name = ProviderConstants.OPERATION_DELETE_EXPUNGE, idempotent = false) @@ -44,6 +46,7 @@ public class DeleteExpungeProvider extends BaseMultiUrlProcessor { RequestDetails theRequestDetails ) { List urls = theUrlsToDeleteExpunge.stream().map(IPrimitiveType::getValue).collect(Collectors.toList()); - return super.processUrls(urls, getBatchSize(theBatchSize), theRequestDetails); + Integer batchSize = myMultiUrlProcessor.getBatchSize(theBatchSize); + return myMultiUrlProcessor.processUrls(urls, batchSize, theRequestDetails); } } diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/BaseMultiUrlProcessor.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/MultiUrlProcessor.java similarity index 80% rename from hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/BaseMultiUrlProcessor.java rename to hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/MultiUrlProcessor.java index ea54a3ca693..6c55631f2c3 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/BaseMultiUrlProcessor.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/MultiUrlProcessor.java @@ -34,20 +34,20 @@ import javax.annotation.Nullable; import java.math.BigDecimal; import java.util.List; -public class BaseMultiUrlProcessor { - protected final FhirContext myFhirContext; +public class MultiUrlProcessor { + private final FhirContext myFhirContext; private final IMultiUrlJobSubmitter myMultiUrlProcessorJobSubmitter; - public BaseMultiUrlProcessor(FhirContext theFhirContext, IMultiUrlJobSubmitter theMultiUrlProcessorJobSubmitter) { + public MultiUrlProcessor(FhirContext theFhirContext, IMultiUrlJobSubmitter theMultiUrlProcessorJobSubmitter) { myMultiUrlProcessorJobSubmitter = theMultiUrlProcessorJobSubmitter; myFhirContext = theFhirContext; } - protected IBaseParameters processUrls(List theUrlsToProcess, Integer theBatchSize, RequestDetails theRequestDetails) { + public IBaseParameters processUrls(List theUrlsToProcess, Integer theBatchSize, RequestDetails theRequestDetails) { try { JobExecution jobExecution = myMultiUrlProcessorJobSubmitter.submitJob(theBatchSize, theUrlsToProcess, theRequestDetails); IBaseParameters retval = ParametersUtil.newInstance(myFhirContext); - ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_DELETE_EXPUNGE_RESPONSE_JOB_ID, jobExecution.getJobId()); + ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, jobExecution.getJobId()); return retval; } catch (JobParametersInvalidException e) { throw new InvalidRequestException("Invalid job parameters: " + e.getMessage(), e); @@ -55,7 +55,7 @@ public class BaseMultiUrlProcessor { } @Nullable - protected Integer getBatchSize(IPrimitiveType theBatchSize) { + public Integer getBatchSize(IPrimitiveType theBatchSize) { Integer batchSize = null; if (theBatchSize != null && !theBatchSize.isEmpty()) { batchSize = theBatchSize.getValue().intValue(); diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java index ffbe7d04e64..f9296160b36 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ProviderConstants.java @@ -84,21 +84,19 @@ public class ProviderConstants { public static final String MDM_DUPLICATE_GOLDEN_RESOURCES = "$mdm-duplicate-golden-resources"; public static final String MDM_NOT_DUPLICATE = "$mdm-not-duplicate"; - public static final String MDM_CLEAR = "$mdm-clear"; - public static final String MDM_CLEAR_SOURCE_TYPE = "sourceType"; + public static final String OPERATION_MDM_CLEAR = "$mdm-clear"; + public static final String OPERATION_MDM_CLEAR_RESOURCE_NAME = "resourceType"; + public static final String OPERATION_MDM_CLEAR_BATCH_SIZE = "batchSize"; public static final String OPERATION_MDM_SUBMIT = "$mdm-submit"; - public static final String MDM_BATCH_RUN_CRITERIA = "criteria" ; - public static final String OPERATION_MDM_BATCH_RUN_OUT_PARAM_SUBMIT_COUNT = "submitted" ; - public static final String OPERATION_MDM_CLEAR_OUT_PARAM_DELETED_COUNT = "deleted"; + public static final String MDM_BATCH_RUN_CRITERIA = "criteria"; public static final String MDM_BATCH_RUN_RESOURCE_TYPE = "resourceType"; - /** * CQL Operations */ public static final String CQL_EVALUATE_MEASURE = "$evaluate-measure"; /** - * Operation name for the $meta operation + * Operation name for the $meta operation */ public static final String OPERATION_META = "$meta"; @@ -146,7 +144,7 @@ public class ProviderConstants { /** * The Spring Batch job id of the delete expunge job created by a $delete-expunge operation */ - public static final String OPERATION_DELETE_EXPUNGE_RESPONSE_JOB_ID = "jobId"; + public static final String OPERATION_BATCH_RESPONSE_JOB_ID = "jobId"; /** * Operation name for the $delete-expunge operation diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ReindexProvider.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ReindexProvider.java index 23c57be0b3a..6d6919f889c 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ReindexProvider.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/ReindexProvider.java @@ -37,11 +37,14 @@ import java.math.BigDecimal; import java.util.List; import java.util.stream.Collectors; -public class ReindexProvider extends BaseMultiUrlProcessor { +public class ReindexProvider { + private final FhirContext myFhirContext; private final IReindexJobSubmitter myReindexJobSubmitter; + private final MultiUrlProcessor myMultiUrlProcessor; public ReindexProvider(FhirContext theFhirContext, IReindexJobSubmitter theReindexJobSubmitter) { - super(theFhirContext, theReindexJobSubmitter); + myFhirContext = theFhirContext; + myMultiUrlProcessor = new MultiUrlProcessor(theFhirContext, theReindexJobSubmitter); myReindexJobSubmitter = theReindexJobSubmitter; } @@ -53,12 +56,12 @@ public class ReindexProvider extends BaseMultiUrlProcessor { RequestDetails theRequestDetails ) { Boolean everything = theEverything != null && theEverything.getValue(); - @Nullable Integer batchSize = getBatchSize(theBatchSize); + @Nullable Integer batchSize = myMultiUrlProcessor.getBatchSize(theBatchSize); if (everything) { return processEverything(batchSize, theRequestDetails); } else if (theUrlsToReindex != null && !theUrlsToReindex.isEmpty()) { List urls = theUrlsToReindex.stream().map(IPrimitiveType::getValue).collect(Collectors.toList()); - return super.processUrls(urls, batchSize, theRequestDetails); + return myMultiUrlProcessor.processUrls(urls, batchSize, theRequestDetails); } else { throw new InvalidRequestException(ProviderConstants.OPERATION_REINDEX + " must specify either everything=true or provide at least one value for " + ProviderConstants.OPERATION_REINDEX_PARAM_URL); } @@ -68,7 +71,7 @@ public class ReindexProvider extends BaseMultiUrlProcessor { try { JobExecution jobExecution = myReindexJobSubmitter.submitEverythingJob(theBatchSize, theRequestDetails); IBaseParameters retval = ParametersUtil.newInstance(myFhirContext); - ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_DELETE_EXPUNGE_RESPONSE_JOB_ID, jobExecution.getJobId()); + ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, jobExecution.getJobId()); return retval; } catch (JobParametersInvalidException e) { throw new InvalidRequestException("Invalid job parameters: " + e.getMessage(), e); diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml index 9e99e253301..5a9a9c59259 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml index add47835990..3094d404947 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT hapi-fhir-spring-boot-sample-client-apache diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml index c447df7668e..92f24d8e8b2 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT hapi-fhir-spring-boot-sample-client-okhttp diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml index 526933dc09e..62d7edf17ed 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT hapi-fhir-spring-boot-sample-server-jersey diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml index 6bcafa300e8..d98baa42dc8 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT hapi-fhir-spring-boot-samples diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml index 93232ddab59..b8e4c3aedce 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/pom.xml b/hapi-fhir-spring-boot/pom.xml index a04306dfeb6..fb7d126c0e1 100644 --- a/hapi-fhir-spring-boot/pom.xml +++ b/hapi-fhir-spring-boot/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-structures-dstu2.1/pom.xml b/hapi-fhir-structures-dstu2.1/pom.xml index f0f14d0d96e..d231b68390e 100644 --- a/hapi-fhir-structures-dstu2.1/pom.xml +++ b/hapi-fhir-structures-dstu2.1/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu2/pom.xml b/hapi-fhir-structures-dstu2/pom.xml index b007972e5e4..fb145734f63 100644 --- a/hapi-fhir-structures-dstu2/pom.xml +++ b/hapi-fhir-structures-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu3/pom.xml b/hapi-fhir-structures-dstu3/pom.xml index b96f7bb77eb..4bedefad1aa 100644 --- a/hapi-fhir-structures-dstu3/pom.xml +++ b/hapi-fhir-structures-dstu3/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-hl7org-dstu2/pom.xml b/hapi-fhir-structures-hl7org-dstu2/pom.xml index 4530c889d95..6130d8e82ac 100644 --- a/hapi-fhir-structures-hl7org-dstu2/pom.xml +++ b/hapi-fhir-structures-hl7org-dstu2/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4/pom.xml b/hapi-fhir-structures-r4/pom.xml index 547c81b936a..2c5394bf890 100644 --- a/hapi-fhir-structures-r4/pom.xml +++ b/hapi-fhir-structures-r4/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/rest/server/helper/BatchHelperR4.java b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/rest/server/helper/BatchHelperR4.java new file mode 100644 index 00000000000..f13bb01f8ca --- /dev/null +++ b/hapi-fhir-structures-r4/src/main/java/org/hl7/fhir/r4/hapi/rest/server/helper/BatchHelperR4.java @@ -0,0 +1,16 @@ +package org.hl7.fhir.r4.hapi.rest.server.helper; + +import ca.uhn.fhir.rest.server.provider.ProviderConstants; +import org.hl7.fhir.r4.model.DecimalType; +import org.hl7.fhir.r4.model.Parameters; + +import javax.annotation.Nonnull; + +public class BatchHelperR4 { + + @Nonnull + public static Long jobIdFromParameters(Parameters response) { + DecimalType jobIdDecimal = (DecimalType) response.getParameter(ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID); + return jobIdDecimal.getValue().longValue(); + } +} diff --git a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java index a5730264c75..df7d54d23e0 100644 --- a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java +++ b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java @@ -5,6 +5,7 @@ import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; import ca.uhn.fhir.rest.api.server.storage.IReindexJobSubmitter; import ca.uhn.fhir.rest.server.BaseR4ServerTest; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; import org.hl7.fhir.r4.model.BooleanType; import org.hl7.fhir.r4.model.DecimalType; import org.hl7.fhir.r4.model.Parameters; @@ -69,8 +70,7 @@ public class BatchProviderTest extends BaseR4ServerTest { .execute(); ourLog.info(myCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(response)); - DecimalType jobId = (DecimalType) response.getParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_RESPONSE_JOB_ID); - assertEquals(TEST_JOB_ID, jobId.getValue().longValue()); + assertEquals(TEST_JOB_ID, BatchHelperR4.jobIdFromParameters(response)); assertThat(myDeleteExpungeJobSubmitter.calledWithUrls, hasSize(2)); assertEquals(url1, myDeleteExpungeJobSubmitter.calledWithUrls.get(0)); assertEquals(url2, myDeleteExpungeJobSubmitter.calledWithUrls.get(1)); diff --git a/hapi-fhir-structures-r5/pom.xml b/hapi-fhir-structures-r5/pom.xml index 4400f91fe15..f2eca4d974b 100644 --- a/hapi-fhir-structures-r5/pom.xml +++ b/hapi-fhir-structures-r5/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-test-utilities/pom.xml b/hapi-fhir-test-utilities/pom.xml index a7004891daf..d0f8df7fe7d 100644 --- a/hapi-fhir-test-utilities/pom.xml +++ b/hapi-fhir-test-utilities/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-testpage-overlay/pom.xml b/hapi-fhir-testpage-overlay/pom.xml index 3d4ecc1f4e4..b38c453f4f7 100644 --- a/hapi-fhir-testpage-overlay/pom.xml +++ b/hapi-fhir-testpage-overlay/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-validation-resources-dstu2.1/pom.xml b/hapi-fhir-validation-resources-dstu2.1/pom.xml index 972e01c812c..0fe4837d110 100644 --- a/hapi-fhir-validation-resources-dstu2.1/pom.xml +++ b/hapi-fhir-validation-resources-dstu2.1/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu2/pom.xml b/hapi-fhir-validation-resources-dstu2/pom.xml index a0956c5ddb3..1fcaa23f8d6 100644 --- a/hapi-fhir-validation-resources-dstu2/pom.xml +++ b/hapi-fhir-validation-resources-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu3/pom.xml b/hapi-fhir-validation-resources-dstu3/pom.xml index c7a59822801..16a4b8587a3 100644 --- a/hapi-fhir-validation-resources-dstu3/pom.xml +++ b/hapi-fhir-validation-resources-dstu3/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r4/pom.xml b/hapi-fhir-validation-resources-r4/pom.xml index 8457e1aebf8..ff9a3a78873 100644 --- a/hapi-fhir-validation-resources-r4/pom.xml +++ b/hapi-fhir-validation-resources-r4/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r5/pom.xml b/hapi-fhir-validation-resources-r5/pom.xml index 3467c240d86..9c17f5a0166 100644 --- a/hapi-fhir-validation-resources-r5/pom.xml +++ b/hapi-fhir-validation-resources-r5/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation/pom.xml b/hapi-fhir-validation/pom.xml index 50945ea7032..e4b063f26f2 100644 --- a/hapi-fhir-validation/pom.xml +++ b/hapi-fhir-validation/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-tinder-plugin/pom.xml b/hapi-tinder-plugin/pom.xml index 51bffb211be..b9efe893439 100644 --- a/hapi-tinder-plugin/pom.xml +++ b/hapi-tinder-plugin/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml @@ -58,37 +58,37 @@ ca.uhn.hapi.fhir hapi-fhir-structures-dstu3 - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-structures-hl7org-dstu2 - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-structures-r4 - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-structures-r5 - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-validation-resources-dstu2 - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-validation-resources-dstu3 - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-validation-resources-r4 - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT org.apache.velocity diff --git a/hapi-tinder-test/pom.xml b/hapi-tinder-test/pom.xml index 86353fff4ab..6db23c691f5 100644 --- a/hapi-tinder-test/pom.xml +++ b/hapi-tinder-test/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 5c72a01da01..9f3e86e83f0 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir pom - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT HAPI-FHIR An open-source implementation of the FHIR specification in Java. https://hapifhir.io diff --git a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml index 7bc3167aba0..a8d1bc42c51 100644 --- a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml +++ b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-client/pom.xml b/tests/hapi-fhir-base-test-mindeps-client/pom.xml index baf9d3f29a7..157a22a3da7 100644 --- a/tests/hapi-fhir-base-test-mindeps-client/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-server/pom.xml b/tests/hapi-fhir-base-test-mindeps-server/pom.xml index aa020297fb3..be74e99f134 100644 --- a/tests/hapi-fhir-base-test-mindeps-server/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.6.0-PRE2-SNAPSHOT + 5.6.0-PRE3-SNAPSHOT ../../pom.xml

sourceTyperesourceType String0..* + The Source resource types you would like to clear. If omitted, all resource types will be cleared. +
batchSizeInteger 0..1 - The Source Resource type you would like to clear. If omitted, will operate over all links. + The number of links that should be deleted at a time. If ommitted, then the batch size will be determined by the value +of [Expunge Batch Size](/apidocs/hapi-fhir-jpaserver-api/ca/uhn/fhir/jpa/api/config/DaoConfig.html#getExpungeBatchSize()) +property.