From 0526da080bda20fe8a06404a33dda25321e221ee Mon Sep 17 00:00:00 2001 From: Ken Stevens Date: Sat, 9 Jul 2022 17:25:55 -0400 Subject: [PATCH] 3757 move delete expunge to batch2 (#3759) * begin with failing test * begin with failing test * remove unused Spring Batch classes. convert service classes * move provider * just one test left to fix * fix test * msg.code * change log * bumping hapi to PRE12 * fix test * review feedback * license update * fix intermittent and status update atomicity * restore url-level partition selection * fix tests Co-authored-by: Ken Stevens --- hapi-deployable-pom/pom.xml | 2 +- hapi-fhir-android/pom.xml | 2 +- hapi-fhir-base/pom.xml | 2 +- .../src/main/java/ca/uhn/fhir/i18n/Msg.java | 2 +- hapi-fhir-batch/pom.xml | 2 +- .../jpa/batch/api/IBatchJobSubmitter.java | 2 +- hapi-fhir-bom/pom.xml | 4 +- hapi-fhir-checkstyle/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-api/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-app/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml | 2 +- hapi-fhir-cli/pom.xml | 2 +- hapi-fhir-client-okhttp/pom.xml | 2 +- hapi-fhir-client/pom.xml | 2 +- hapi-fhir-converter/pom.xml | 2 +- hapi-fhir-dist/pom.xml | 2 +- hapi-fhir-docs/pom.xml | 2 +- .../6_1_0/3757-delete-expunge-batch2.yaml | 4 + hapi-fhir-jacoco/pom.xml | 2 +- hapi-fhir-jaxrsserver-base/pom.xml | 2 +- hapi-fhir-jpa/pom.xml | 2 +- .../CircularQueueCaptureQueriesListener.java | 2 +- hapi-fhir-jpaserver-base/pom.xml | 2 +- .../uhn/fhir/jpa/batch/BatchJobsConfig.java | 4 - .../fhir/jpa/batch/CommonBatchJobConfig.java | 29 --- .../job/MultiUrlJobParameterValidator.java | 67 ----- .../batch/job/PartitionedUrlValidator.java | 75 ------ .../jpa/batch/job/model/PartitionedUrl.java | 61 ----- .../jpa/batch/job/model/RequestListJson.java | 82 ------ .../listener/PidReaderCounterListener.java | 48 ---- .../MdmBatchJobSubmitterFactoryImpl.java | 35 --- .../mdm/batch/MdmClearJobSubmitterImpl.java | 85 ------- .../mdm/batch/job/MdmClearJobConfig.java | 124 --------- .../batch/mdm/batch/job/MdmLinkDeleter.java | 85 ------- ...erseCronologicalBatchMdmLinkPidReader.java | 59 ----- ...BaseReverseCronologicalBatchPidReader.java | 240 ------------------ .../reader/BatchDateThresholdUpdater.java | 96 ------- .../batch/reader/BatchResourceSearcher.java | 65 ----- ...CronologicalBatchAllResourcePidReader.java | 163 ------------ ...rseCronologicalBatchResourcePidReader.java | 71 ------ .../jpa/batch/writer/SqlExecutorWriter.java | 69 ----- .../fhir/jpa/config/Batch2SupportConfig.java | 52 ++++ .../ca/uhn/fhir/jpa/config/JpaConfig.java | 42 +-- .../fhir/jpa/dao/BaseHapiFhirResourceDao.java | 8 +- .../DeleteExpungeSqlBuilder.java} | 58 ++--- .../delete/batch2/DeleteExpungeSvcImpl.java | 60 +++++ .../delete/job/DeleteExpungeJobConfig.java | 102 -------- ...ndexSvcImpl.java => Batch2DaoSvcImpl.java} | 4 +- hapi-fhir-jpaserver-cql/pom.xml | 2 +- hapi-fhir-jpaserver-mdm/pom.xml | 2 +- .../fhir/jpa/mdm/config/MdmCommonConfig.java | 6 - .../jpa/mdm/svc/MdmControllerSvcImpl.java | 3 - .../jpa/mdm/provider/BaseProviderR4Test.java | 3 - hapi-fhir-jpaserver-model/pom.xml | 2 +- hapi-fhir-jpaserver-searchparam/pom.xml | 2 +- hapi-fhir-jpaserver-subscription/pom.xml | 2 +- hapi-fhir-jpaserver-test-utilities/pom.xml | 2 +- .../ca/uhn/fhir/jpa/test/Batch2JobHelper.java | 42 ++- .../batch/job/MultiUrlJobParameterUtil.java | 29 --- .../jpa/batch/mdm/job/MdmLinkDeleterTest.java | 64 ----- .../reader/BatchDateThresholdUpdaterTest.java | 125 --------- ...ronologicalBatchResourcePidReaderTest.java | 173 ------------- .../jpa/dao/expunge/DeleteExpungeDaoTest.java | 66 +++-- .../jpa/delete/job/DeleteExpungeJobTest.java | 47 ++-- .../MultiUrlJobParameterValidatorTest.java | 73 ------ .../provider/DeleteExpungeProviderTest.java | 2 +- .../r4/BaseResourceProviderR4Test.java | 6 +- .../r4/MultitenantBatchOperationR4Test.java | 19 +- .../jpa/provider/r4/SystemProviderR4Test.java | 39 +-- .../reindex/ResourceReindexSvcImplTest.java | 4 +- .../fhir/jpa/stresstest/StressTestR4Test.java | 15 +- hapi-fhir-jpaserver-uhnfhirtest/pom.xml | 2 +- hapi-fhir-server-mdm/pom.xml | 2 +- hapi-fhir-server-openapi/pom.xml | 2 +- hapi-fhir-server/pom.xml | 2 +- .../storage/IDeleteExpungeJobSubmitter.java | 12 +- .../server/storage/IMultiUrlJobSubmitter.java | 37 --- .../server/storage/IReindexJobSubmitter.java | 30 --- .../server/provider/MultiUrlProcessor.java | 66 ----- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../hapi-fhir-spring-boot-samples/pom.xml | 2 +- .../hapi-fhir-spring-boot-starter/pom.xml | 2 +- hapi-fhir-spring-boot/pom.xml | 2 +- hapi-fhir-sql-migrate/pom.xml | 2 +- hapi-fhir-storage-batch2-jobs/pom.xml | 2 +- .../batch2/jobs/config/Batch2JobsConfig.java | 5 +- .../batch2/jobs/config/BatchCommonCtx.java | 13 + .../jobs/expunge/DeleteExpungeAppCtx.java | 99 ++++++++ .../expunge/DeleteExpungeJobParameters.java | 9 +- .../DeleteExpungeJobParametersValidator.java | 42 +++ .../DeleteExpungeJobSubmitterImpl.java | 63 +++-- .../jobs/expunge}/DeleteExpungeProvider.java | 32 ++- .../jobs/expunge/DeleteExpungeStep.java | 108 ++++++++ .../batch2/jobs/reindex/ReindexAppCtx.java | 28 +- .../jobs/reindex/ReindexIdChunkProducer.java | 49 ---- .../jobs/reindex/ReindexJobParameters.java | 31 +-- .../ReindexJobParametersValidator.java | 26 +- .../batch2/jobs/reindex/ReindexProvider.java | 22 +- .../fhir/batch2/jobs/BaseR4ServerTest.java | 45 ++++ .../expunge/DeleteExpungeProviderTest.java | 80 ++++++ .../ReindexJobParametersValidatorTest.java | 41 --- .../jobs/reindex/ReindexProviderTest.java | 17 +- hapi-fhir-storage-batch2/pom.xml | 2 +- .../fhir/batch2/coordinator/JobDataSink.java | 4 + .../chunk/PartitionedUrlChunkRangeJson.java | 20 +- .../jobs/parameters/PartitionedUrl.java | 30 +++ .../PartitionedUrlListJobParameters.java | 56 ++++ .../jobs/parameters/UrlListValidator.java | 56 ++++ .../jobs/parameters/UrlPartitioner.java | 27 ++ .../jobs/step}/GenerateRangeChunksStep.java | 25 +- .../fhir/batch2/jobs/step}/LoadIdsStep.java | 24 +- .../PartitionedUrlListIdChunkProducer.java | 55 ++++ .../UrlListParametersValidatorTest.java | 48 ++++ .../batch2/jobs/step}/LoadIdsStepTest.java | 22 +- hapi-fhir-storage-mdm/pom.xml | 2 +- hapi-fhir-storage-test-utilities/pom.xml | 2 +- hapi-fhir-storage/pom.xml | 2 +- ...urceReindexSvc.java => IBatch2DaoSvc.java} | 2 +- .../fhir/jpa/api/svc/IDeleteExpungeSvc.java | 18 +- hapi-fhir-structures-dstu2.1/pom.xml | 2 +- hapi-fhir-structures-dstu2/pom.xml | 2 +- hapi-fhir-structures-dstu3/pom.xml | 2 +- hapi-fhir-structures-hl7org-dstu2/pom.xml | 2 +- hapi-fhir-structures-r4/pom.xml | 2 +- .../server/provider/BatchProviderTest.java | 116 --------- hapi-fhir-structures-r5/pom.xml | 2 +- hapi-fhir-test-utilities/pom.xml | 2 +- hapi-fhir-testpage-overlay/pom.xml | 2 +- .../pom.xml | 2 +- hapi-fhir-validation-resources-dstu2/pom.xml | 2 +- hapi-fhir-validation-resources-dstu3/pom.xml | 2 +- hapi-fhir-validation-resources-r4/pom.xml | 2 +- hapi-fhir-validation-resources-r5/pom.xml | 2 +- hapi-fhir-validation/pom.xml | 2 +- hapi-tinder-plugin/pom.xml | 16 +- hapi-tinder-test/pom.xml | 2 +- pom.xml | 4 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- 143 files changed, 1225 insertions(+), 2785 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3757-delete-expunge-batch2.yaml delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterValidator.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/PartitionedUrl.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/RequestListJson.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/listener/PidReaderCounterListener.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmBatchJobSubmitterFactoryImpl.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmClearJobSubmitterImpl.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmClearJobConfig.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmLinkDeleter.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/ReverseCronologicalBatchMdmLinkPidReader.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdater.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchResourceSearcher.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/writer/SqlExecutorWriter.java create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/Batch2SupportConfig.java rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/{job/DeleteExpungeProcessor.java => batch2/DeleteExpungeSqlBuilder.java} (76%) create mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java delete mode 100644 hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java rename hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/{ResourceReindexSvcImpl.java => Batch2DaoSvcImpl.java} (97%) delete mode 100644 hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterUtil.java delete mode 100644 hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleterTest.java delete mode 100644 hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdaterTest.java delete mode 100644 hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReaderTest.java delete mode 100644 hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/MultiUrlJobParameterValidatorTest.java delete mode 100644 hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IMultiUrlJobSubmitter.java delete mode 100644 hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IReindexJobSubmitter.java delete mode 100644 hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/MultiUrlProcessor.java create mode 100644 hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/BatchCommonCtx.java create mode 100644 hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeAppCtx.java rename hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java => hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParameters.java (73%) create mode 100644 hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParametersValidator.java rename {hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete => hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge}/DeleteExpungeJobSubmitterImpl.java (53%) rename {hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider => hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge}/DeleteExpungeProvider.java (62%) create mode 100644 hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java delete mode 100644 hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexIdChunkProducer.java create mode 100644 hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/BaseR4ServerTest.java create mode 100644 hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeProviderTest.java delete mode 100644 hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidatorTest.java rename hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexChunkRangeJson.java => hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/PartitionedUrlChunkRangeJson.java (61%) create mode 100644 hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrl.java create mode 100644 hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrlListJobParameters.java create mode 100644 hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListValidator.java create mode 100644 hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlPartitioner.java rename {hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex => hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step}/GenerateRangeChunksStep.java (61%) rename {hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex => hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step}/LoadIdsStep.java (52%) create mode 100644 hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/PartitionedUrlListIdChunkProducer.java create mode 100644 hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListParametersValidatorTest.java rename {hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex => hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/step}/LoadIdsStepTest.java (77%) rename hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/{IResourceReindexSvc.java => IBatch2DaoSvc.java} (98%) rename hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmClearJobSubmitter.java => hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IDeleteExpungeSvc.java (59%) delete mode 100644 hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java diff --git a/hapi-deployable-pom/pom.xml b/hapi-deployable-pom/pom.xml index 3b3970107e7..16ad3b1ed5f 100644 --- a/hapi-deployable-pom/pom.xml +++ b/hapi-deployable-pom/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-android/pom.xml b/hapi-fhir-android/pom.xml index bd2caced4a3..a917a6a9b49 100644 --- a/hapi-fhir-android/pom.xml +++ b/hapi-fhir-android/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/pom.xml b/hapi-fhir-base/pom.xml index db38d2480a4..03f536c2673 100644 --- a/hapi-fhir-base/pom.xml +++ b/hapi-fhir-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java index 5ac5254823b..cdab93ffa58 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/i18n/Msg.java @@ -25,7 +25,7 @@ public final class Msg { /** * IMPORTANT: Please update the following comment after you add a new code - * Last code value: 2100 + * Last code value: 2101 */ private Msg() {} diff --git a/hapi-fhir-batch/pom.xml b/hapi-fhir-batch/pom.xml index c278af4d8a9..cef986c13ae 100644 --- a/hapi-fhir-batch/pom.xml +++ b/hapi-fhir-batch/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-batch/src/main/java/ca/uhn/fhir/jpa/batch/api/IBatchJobSubmitter.java b/hapi-fhir-batch/src/main/java/ca/uhn/fhir/jpa/batch/api/IBatchJobSubmitter.java index 9b20f158576..efc95c6f5b2 100644 --- a/hapi-fhir-batch/src/main/java/ca/uhn/fhir/jpa/batch/api/IBatchJobSubmitter.java +++ b/hapi-fhir-batch/src/main/java/ca/uhn/fhir/jpa/batch/api/IBatchJobSubmitter.java @@ -26,7 +26,7 @@ import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersInvalidException; /** - * @deprecated we are in the process of converting to batch2 + * @deprecated use IJobCoordinator instead */ @Deprecated public interface IBatchJobSubmitter { diff --git a/hapi-fhir-bom/pom.xml b/hapi-fhir-bom/pom.xml index 79f37ad22c7..55716699704 100644 --- a/hapi-fhir-bom/pom.xml +++ b/hapi-fhir-bom/pom.xml @@ -3,14 +3,14 @@ 4.0.0 ca.uhn.hapi.fhir hapi-fhir-bom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT pom HAPI FHIR BOM ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-checkstyle/pom.xml b/hapi-fhir-checkstyle/pom.xml index 5d2b1cf2c69..ed67dcb426e 100644 --- a/hapi-fhir-checkstyle/pom.xml +++ b/hapi-fhir-checkstyle/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml index 3505e68b538..79e7058fcf1 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml index 0083658941e..84eec664242 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir-cli - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml index c4751e261d2..4a3469fb0b3 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../../hapi-deployable-pom diff --git a/hapi-fhir-cli/pom.xml b/hapi-fhir-cli/pom.xml index 2469fbc253a..7921f8a1915 100644 --- a/hapi-fhir-cli/pom.xml +++ b/hapi-fhir-cli/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-client-okhttp/pom.xml b/hapi-fhir-client-okhttp/pom.xml index 96389e525f1..93d133cd14a 100644 --- a/hapi-fhir-client-okhttp/pom.xml +++ b/hapi-fhir-client-okhttp/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-client/pom.xml b/hapi-fhir-client/pom.xml index 735413ca850..96461cda1f3 100644 --- a/hapi-fhir-client/pom.xml +++ b/hapi-fhir-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-converter/pom.xml b/hapi-fhir-converter/pom.xml index 1ed7701885f..eb29b7769e8 100644 --- a/hapi-fhir-converter/pom.xml +++ b/hapi-fhir-converter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-dist/pom.xml b/hapi-fhir-dist/pom.xml index c6010d81016..c4ad8056e1c 100644 --- a/hapi-fhir-dist/pom.xml +++ b/hapi-fhir-dist/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-docs/pom.xml b/hapi-fhir-docs/pom.xml index 3d8fb140f64..ef131eadade 100644 --- a/hapi-fhir-docs/pom.xml +++ b/hapi-fhir-docs/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3757-delete-expunge-batch2.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3757-delete-expunge-batch2.yaml new file mode 100644 index 00000000000..9b9c9108cd5 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_1_0/3757-delete-expunge-batch2.yaml @@ -0,0 +1,4 @@ +--- +type: change +issue: 3757 +title: "The Delete Expunge operation has been moved from Spring Batch to Batch 2." diff --git a/hapi-fhir-jacoco/pom.xml b/hapi-fhir-jacoco/pom.xml index ca8fb92c2c7..42bf21e0eaf 100644 --- a/hapi-fhir-jacoco/pom.xml +++ b/hapi-fhir-jacoco/pom.xml @@ -11,7 +11,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jaxrsserver-base/pom.xml b/hapi-fhir-jaxrsserver-base/pom.xml index 94ea1b90f4b..cf11493972a 100644 --- a/hapi-fhir-jaxrsserver-base/pom.xml +++ b/hapi-fhir-jaxrsserver-base/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpa/pom.xml b/hapi-fhir-jpa/pom.xml index 40058d1db1c..d3c8bc4cb46 100644 --- a/hapi-fhir-jpa/pom.xml +++ b/hapi-fhir-jpa/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/util/CircularQueueCaptureQueriesListener.java b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/util/CircularQueueCaptureQueriesListener.java index d549f970587..00c97e40ff1 100644 --- a/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/util/CircularQueueCaptureQueriesListener.java +++ b/hapi-fhir-jpa/src/main/java/ca/uhn/fhir/jpa/util/CircularQueueCaptureQueriesListener.java @@ -396,7 +396,7 @@ public class CircularQueueCaptureQueriesListener extends BaseCaptureQueriesListe public int countDeleteQueries() { return getDeleteQueries() .stream() - .map(t->t.getSize()) + .map(SqlQuery::getSize) .reduce(0, Integer::sum); } diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index f91bdf8868b..c284b8cd694 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java index 4389fc6ab76..1d17dca075f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/BatchJobsConfig.java @@ -21,12 +21,10 @@ package ca.uhn.fhir.jpa.batch; */ import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; -import ca.uhn.fhir.jpa.batch.mdm.batch.job.MdmClearJobConfig; import ca.uhn.fhir.jpa.batch.svc.BatchJobSubmitterImpl; import ca.uhn.fhir.jpa.bulk.export.job.BulkExportJobConfig; import ca.uhn.fhir.jpa.bulk.imprt.job.BulkImportJobConfig; import ca.uhn.fhir.jpa.config.BatchJobRegisterer; -import ca.uhn.fhir.jpa.delete.job.DeleteExpungeJobConfig; import ca.uhn.fhir.jpa.term.job.TermCodeSystemDeleteJobConfig; import ca.uhn.fhir.jpa.term.job.TermCodeSystemVersionDeleteJobConfig; import org.springframework.batch.core.configuration.JobRegistry; @@ -45,8 +43,6 @@ import org.springframework.context.annotation.Import; CommonBatchJobConfig.class, BulkExportJobConfig.class, BulkImportJobConfig.class, - DeleteExpungeJobConfig.class, - MdmClearJobConfig.class, TermCodeSystemDeleteJobConfig.class, TermCodeSystemVersionDeleteJobConfig.class // When you define a new batch job, add it here. diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java index f4821bea2dd..e7f4f401448 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/CommonBatchJobConfig.java @@ -20,14 +20,8 @@ package ca.uhn.fhir.jpa.batch; * #L% */ -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; -import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; import ca.uhn.fhir.jpa.batch.processor.GoldenResourceAnnotatingProcessor; import ca.uhn.fhir.jpa.batch.processor.PidToIBaseResourceProcessor; -import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; -import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -36,29 +30,6 @@ import org.springframework.context.annotation.Configuration; public class CommonBatchJobConfig { public static final int MINUTES_IN_FUTURE_TO_PROCESS_FROM = 1; - @Bean - public MultiUrlJobParameterValidator multiUrlProcessorParameterValidator(MatchUrlService theMatchUrlService, DaoRegistry theDaoRegistry) { - return new MultiUrlJobParameterValidator(theMatchUrlService, theDaoRegistry); - } - - @Bean - @StepScope - public SqlExecutorWriter sqlExecutorWriter() { - return new SqlExecutorWriter(); - } - - @Bean - @StepScope - public PidReaderCounterListener pidCountRecorderListener() { - return new PidReaderCounterListener(); - } - - @Bean - @StepScope - public ReverseCronologicalBatchResourcePidReader reverseCronologicalBatchResourcePidReader() { - return new ReverseCronologicalBatchResourcePidReader(); - } - @Bean @StepScope public PidToIBaseResourceProcessor pidToResourceProcessor() { diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterValidator.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterValidator.java deleted file mode 100644 index 05b36db48a1..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterValidator.java +++ /dev/null @@ -1,67 +0,0 @@ -package ca.uhn.fhir.jpa.batch.job; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.batch.job.model.PartitionedUrl; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersInvalidException; -import org.springframework.batch.core.JobParametersValidator; - -/** - * This class will prevent a job from running any of the provided URLs are not valid on this server. - */ -public class MultiUrlJobParameterValidator implements JobParametersValidator { - public static String JOB_PARAM_OPERATION_NAME = "operation-name"; - private final MatchUrlService myMatchUrlService; - private final DaoRegistry myDaoRegistry; - - public MultiUrlJobParameterValidator(MatchUrlService theMatchUrlService, DaoRegistry theDaoRegistry) { - myMatchUrlService = theMatchUrlService; - myDaoRegistry = theDaoRegistry; - } - - @Override - public void validate(JobParameters theJobParameters) throws JobParametersInvalidException { - if (theJobParameters == null) { - throw new JobParametersInvalidException(Msg.code(1280) + "This job requires Parameters: [urlList]"); - } - - RequestListJson requestListJson = RequestListJson.fromJson(theJobParameters.getString(BatchConstants.JOB_PARAM_REQUEST_LIST)); - for (PartitionedUrl partitionedUrl : requestListJson.getPartitionedUrls()) { - String url = partitionedUrl.getUrl(); - try { - ResourceSearch resourceSearch = myMatchUrlService.getResourceSearch(url, partitionedUrl.getRequestPartitionId()); - String resourceName = resourceSearch.getResourceName(); - if (!myDaoRegistry.isResourceTypeSupported(resourceName)) { - throw new JobParametersInvalidException(Msg.code(1281) + "The resource type " + resourceName + " is not supported on this server."); - } - } catch (UnsupportedOperationException e) { - throw new JobParametersInvalidException(Msg.code(1282) + "Failed to parse " + theJobParameters.getString(JOB_PARAM_OPERATION_NAME) + " " + BatchConstants.JOB_PARAM_REQUEST_LIST + " item " + url + ": " + e.getMessage()); - } - } - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java deleted file mode 100644 index 10ea1f44326..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/PartitionedUrlValidator.java +++ /dev/null @@ -1,75 +0,0 @@ -package ca.uhn.fhir.jpa.batch.job; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.batch.job.model.PartitionedUrl; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; -import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -public class PartitionedUrlValidator { - @Autowired - MatchUrlService myMatchUrlService; - @Autowired - IRequestPartitionHelperSvc myRequestPartitionHelperSvc; - @Autowired - FhirContext myFhirContext; - - public PartitionedUrlValidator() { - } - - /** - * This method will throw an exception if the user is not allowed to access the requested resource type on the partition determined by the request - */ - - public RequestListJson buildRequestListJson(RequestDetails theRequest, List theUrlsToProcess) { - List partitionedUrls = new ArrayList<>(); - for (String url : theUrlsToProcess) { - ResourceSearch resourceSearch = myMatchUrlService.getResourceSearch(url); - RequestPartitionId requestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequestForSearchType(theRequest, resourceSearch.getResourceName(), resourceSearch.getSearchParameterMap(), null); - partitionedUrls.add(new PartitionedUrl(url, requestPartitionId)); - } - RequestListJson retval = new RequestListJson(); - retval.setPartitionedUrls(partitionedUrls); - return retval; - } - - public RequestPartitionId requestPartitionIdFromRequest(RequestDetails theRequest) { - Set allResourceNames = myFhirContext.getResourceTypes(); - SearchParameterMap map = SearchParameterMap.newSynchronous(); - // Verify that the user has access to every resource type on the server: - for (String resourceName : allResourceNames) { - myRequestPartitionHelperSvc.determineReadPartitionForRequestForSearchType(theRequest, resourceName, map, null); - } - // Then return the partition for the Patient resource type. Note Patient was an arbitrary choice here. - return myRequestPartitionHelperSvc.determineReadPartitionForRequestForSearchType(theRequest, "Patient", map, null); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/PartitionedUrl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/PartitionedUrl.java deleted file mode 100644 index 4516e006cf6..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/PartitionedUrl.java +++ /dev/null @@ -1,61 +0,0 @@ -package ca.uhn.fhir.jpa.batch.job.model; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.model.api.IModelJson; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class PartitionedUrl implements IModelJson { - @JsonProperty("url") - private String myUrl; - - @JsonProperty("requestPartitionId") - private RequestPartitionId myRequestPartitionId; - - public PartitionedUrl() { - } - - public PartitionedUrl(String theUrl, RequestPartitionId theRequestPartitionId) { - myUrl = theUrl; - myRequestPartitionId = theRequestPartitionId; - } - - public String getUrl() { - return myUrl; - } - - public void setUrl(String theUrl) { - myUrl = theUrl; - } - - public RequestPartitionId getRequestPartitionId() { - return myRequestPartitionId; - } - - public void setRequestPartitionId(RequestPartitionId theRequestPartitionId) { - myRequestPartitionId = theRequestPartitionId; - } - - public boolean isPartitioned() { - return myRequestPartitionId != null && !myRequestPartitionId.isDefaultPartition(); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/RequestListJson.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/RequestListJson.java deleted file mode 100644 index f3968c23777..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/job/model/RequestListJson.java +++ /dev/null @@ -1,82 +0,0 @@ -package ca.uhn.fhir.jpa.batch.job.model; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.model.api.IModelJson; -import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; -import ca.uhn.fhir.util.JsonUtil; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.List; - -/** - * Serialize a list of URLs and partition ids so Spring Batch can store it as a String - */ -public class RequestListJson implements IModelJson { - static final ObjectMapper ourObjectMapper = new ObjectMapper(); - - @JsonProperty("partitionedUrls") - private List myPartitionedUrls; - - public static RequestListJson fromUrlStringsAndRequestPartitionIds(List theUrls, List theRequestPartitionIds) { - assert theUrls.size() == theRequestPartitionIds.size(); - - RequestListJson retval = new RequestListJson(); - List partitionedUrls = new ArrayList<>(); - for (int i = 0; i < theUrls.size(); ++i) { - partitionedUrls.add(new PartitionedUrl(theUrls.get(i), theRequestPartitionIds.get(i))); - } - retval.setPartitionedUrls(partitionedUrls); - return retval; - } - - public static RequestListJson fromJson(String theJson) { - try { - return ourObjectMapper.readValue(theJson, RequestListJson.class); - } catch (JsonProcessingException e) { - throw new InternalErrorException(Msg.code(1283) + "Failed to decode " + RequestListJson.class); - } - } - - public String toJson() { - return JsonUtil.serializeOrInvalidRequest(this); - } - - @Override - public String toString() { - return "RequestListJson{" + - "myPartitionedUrls=" + myPartitionedUrls + - '}'; - } - - public List getPartitionedUrls() { - return myPartitionedUrls; - } - - public void setPartitionedUrls(List thePartitionedUrls) { - myPartitionedUrls = thePartitionedUrls; - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/listener/PidReaderCounterListener.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/listener/PidReaderCounterListener.java deleted file mode 100644 index 00adc91f8bc..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/listener/PidReaderCounterListener.java +++ /dev/null @@ -1,48 +0,0 @@ -package ca.uhn.fhir.jpa.batch.listener; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.annotation.AfterRead; -import org.springframework.batch.core.annotation.BeforeStep; - -import java.util.List; - -/** - * Add the number of pids processed to the execution context so we can track progress of the job - */ -public class PidReaderCounterListener { - public static final String RESOURCE_TOTAL_PROCESSED = "resource.total.processed"; - - private StepExecution myStepExecution; - private Long myTotalPidsProcessed = 0L; - - @BeforeStep - public void setStepExecution(StepExecution stepExecution) { - myStepExecution = stepExecution; - } - - @AfterRead - public void afterRead(List thePids) { - myTotalPidsProcessed += thePids.size(); - myStepExecution.getExecutionContext().putLong(RESOURCE_TOTAL_PROCESSED, myTotalPidsProcessed); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmBatchJobSubmitterFactoryImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmBatchJobSubmitterFactoryImpl.java deleted file mode 100644 index 5d64dc8a13c..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmBatchJobSubmitterFactoryImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -package ca.uhn.fhir.jpa.batch.mdm.batch; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory; -import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; -import org.springframework.beans.factory.annotation.Autowired; - -public class MdmBatchJobSubmitterFactoryImpl implements IMdmBatchJobSubmitterFactory { - @Autowired - IMdmClearJobSubmitter myMdmClearJobSubmitter; - - @Override - public IMdmClearJobSubmitter getClearJobSubmitter() { - return myMdmClearJobSubmitter; - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmClearJobSubmitterImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmClearJobSubmitterImpl.java deleted file mode 100644 index 01effe1a705..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/MdmClearJobSubmitterImpl.java +++ /dev/null @@ -1,85 +0,0 @@ -package ca.uhn.fhir.jpa.batch.mdm.batch; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.interceptor.api.HookParams; -import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; -import ca.uhn.fhir.interceptor.api.Pointcut; -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.batch.job.PartitionedUrlValidator; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; -import ca.uhn.fhir.jpa.batch.mdm.batch.job.ReverseCronologicalBatchMdmLinkPidReader; -import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.server.exceptions.ForbiddenOperationException; -import ca.uhn.fhir.rest.server.provider.ProviderConstants; -import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; -import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersInvalidException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; - -import javax.transaction.Transactional; -import java.util.List; - -public class MdmClearJobSubmitterImpl implements IMdmClearJobSubmitter { - @Autowired - DaoConfig myDaoConfig; - @Autowired - PartitionedUrlValidator myPartitionedUrlValidator; - @Autowired - IInterceptorBroadcaster myInterceptorBroadcaster; - @Autowired - private IBatchJobSubmitter myBatchJobSubmitter; - @Autowired - @Qualifier(BatchConstants.MDM_CLEAR_JOB_NAME) - private Job myMdmClearJob; - - @Override - @Transactional(Transactional.TxType.NEVER) - public JobExecution submitJob(Integer theBatchSize, List theUrls, RequestDetails theRequest) throws JobParametersInvalidException { - if (theBatchSize == null) { - theBatchSize = myDaoConfig.getExpungeBatchSize(); - } - if (!myDaoConfig.canDeleteExpunge()) { - throw new ForbiddenOperationException(Msg.code(1278) + "Delete Expunge not allowed: " + myDaoConfig.cannotDeleteExpungeReason()); - } - - RequestListJson requestListJson = myPartitionedUrlValidator.buildRequestListJson(theRequest, theUrls); - - for (String url : theUrls) { - HookParams params = new HookParams() - .add(RequestDetails.class, theRequest) - .addIfMatchesType(ServletRequestDetails.class, theRequest) - .add(String.class, url); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PRE_DELETE_EXPUNGE, params); - } - - JobParameters jobParameters = ReverseCronologicalBatchMdmLinkPidReader.buildJobParameters(ProviderConstants.OPERATION_MDM_CLEAR, theBatchSize, requestListJson); - return myBatchJobSubmitter.runJob(myMdmClearJob, jobParameters); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmClearJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmClearJobConfig.java deleted file mode 100644 index d7a243c9bda..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmClearJobConfig.java +++ /dev/null @@ -1,124 +0,0 @@ -package ca.uhn.fhir.jpa.batch.mdm.batch.job; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; -import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; -import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; -import ca.uhn.fhir.jpa.delete.job.DeleteExpungeProcessor; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepScope; -import org.springframework.batch.core.listener.ExecutionContextPromotionListener; -import org.springframework.batch.item.support.CompositeItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Lazy; - -import java.util.ArrayList; -import java.util.List; - -import static ca.uhn.fhir.jpa.batch.config.BatchConstants.MDM_CLEAR_JOB_NAME; - -/** - * Spring batch Job configuration file. Contains all necessary plumbing to run a - * $mdm-clear job. - */ -@Configuration -public class MdmClearJobConfig { - public static final String MDM_CLEAR_RESOURCE_LIST_STEP_NAME = "mdm-clear-resource-list-step"; - - @Autowired - private StepBuilderFactory myStepBuilderFactory; - @Autowired - private JobBuilderFactory myJobBuilderFactory; - @Autowired - private DeleteExpungeProcessor myDeleteExpungeProcessor; - - @Autowired - @Qualifier("deleteExpungePromotionListener") - private ExecutionContextPromotionListener myDeleteExpungePromotionListener; - - @Autowired - private MultiUrlJobParameterValidator myMultiUrlProcessorParameterValidator; - - @Autowired - private PidReaderCounterListener myPidCountRecorderListener; - - @Autowired - private SqlExecutorWriter mySqlExecutorWriter; - - @Bean(name = MDM_CLEAR_JOB_NAME) - @Lazy - public Job mdmClearJob() { - return myJobBuilderFactory.get(MDM_CLEAR_JOB_NAME) - .validator(myMultiUrlProcessorParameterValidator) - .start(mdmClearUrlListStep()) - .build(); - } - - @Bean - public Step mdmClearUrlListStep() { - return myStepBuilderFactory.get(MDM_CLEAR_RESOURCE_LIST_STEP_NAME) - ., List>chunk(1) - .reader(reverseCronologicalBatchMdmLinkPidReader()) - .processor(deleteThenExpungeCompositeProcessor()) - .writer(mySqlExecutorWriter) - .listener(myPidCountRecorderListener) - .listener(myDeleteExpungePromotionListener) - .build(); - } - - @Bean - @StepScope - public CompositeItemProcessor, List> deleteThenExpungeCompositeProcessor() { - CompositeItemProcessor, List> compositeProcessor = new CompositeItemProcessor<>(); - List itemProcessors = new ArrayList<>(); - itemProcessors.add(mdmLinkDeleter()); - itemProcessors.add(myDeleteExpungeProcessor); - compositeProcessor.setDelegates(itemProcessors); - return compositeProcessor; - } - - @Bean - @StepScope - public ReverseCronologicalBatchMdmLinkPidReader reverseCronologicalBatchMdmLinkPidReader() { - return new ReverseCronologicalBatchMdmLinkPidReader(); - } - - @Bean - public MdmLinkDeleter mdmLinkDeleter() { - return new MdmLinkDeleter(); - } - - @Bean - public ExecutionContextPromotionListener mdmClearPromotionListener() { - ExecutionContextPromotionListener listener = new ExecutionContextPromotionListener(); - - listener.setKeys(new String[]{PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED}); - - return listener; - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmLinkDeleter.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmLinkDeleter.java deleted file mode 100644 index 41f40efb58a..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/MdmLinkDeleter.java +++ /dev/null @@ -1,85 +0,0 @@ -package ca.uhn.fhir.jpa.batch.mdm.batch.job; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.dao.data.IMdmLinkDao; -import ca.uhn.fhir.jpa.dao.expunge.PartitionRunner; -import ca.uhn.fhir.jpa.entity.MdmLink; -import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; -import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.support.TransactionTemplate; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.stream.Collectors; - -/** - * Take MdmLink pids in and output golden resource pids out - */ - -public class MdmLinkDeleter implements ItemProcessor, List> { - public static final String PROCESS_NAME = "MdmClear"; - public static final String THREAD_PREFIX = "mdmClear"; - private static final Logger ourLog = LoggerFactory.getLogger(MdmLinkDeleter.class); - @Autowired - protected PlatformTransactionManager myTxManager; - @Autowired - IMdmLinkDao myMdmLinkDao; - @Autowired - DaoConfig myDaoConfig; - - @Override - public List process(List thePidList) throws Exception { - ConcurrentLinkedQueue goldenPidAggregator = new ConcurrentLinkedQueue<>(); - PartitionRunner partitionRunner = new PartitionRunner(PROCESS_NAME, THREAD_PREFIX, myDaoConfig.getReindexBatchSize(), myDaoConfig.getReindexThreadCount()); - partitionRunner.runInPartitionedThreads(ResourcePersistentId.fromLongList(thePidList), pids -> removeLinks(pids, goldenPidAggregator)); - return new ArrayList<>(goldenPidAggregator); - } - - private void removeLinks(List pidList, ConcurrentLinkedQueue theGoldenPidAggregator) { - TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); - - txTemplate.executeWithoutResult(t -> theGoldenPidAggregator.addAll(deleteMdmLinksAndReturnGoldenResourcePids(ResourcePersistentId.toLongList(pidList)))); - } - - public List deleteMdmLinksAndReturnGoldenResourcePids(List thePids) { - List links = myMdmLinkDao.findAllById(thePids); - Set goldenResources = links.stream().map(MdmLink::getGoldenResourcePid).collect(Collectors.toSet()); - //TODO GGG this is probably invalid... we are essentially looking for GOLDEN -> GOLDEN links, which are either POSSIBLE_DUPLICATE - //and REDIRECT - goldenResources.addAll(links.stream() - .filter(link -> link.getMatchResult().equals(MdmMatchResultEnum.REDIRECT) - || link.getMatchResult().equals(MdmMatchResultEnum.POSSIBLE_DUPLICATE)) - .map(MdmLink::getSourcePid).collect(Collectors.toSet())); - ourLog.info("Deleting {} MDM link records...", links.size()); - myMdmLinkDao.deleteAll(links); - ourLog.info("{} MDM link records deleted", links.size()); - return new ArrayList<>(goldenResources); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/ReverseCronologicalBatchMdmLinkPidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/ReverseCronologicalBatchMdmLinkPidReader.java deleted file mode 100644 index 19f7c23a55b..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/mdm/batch/job/ReverseCronologicalBatchMdmLinkPidReader.java +++ /dev/null @@ -1,59 +0,0 @@ -package ca.uhn.fhir.jpa.batch.mdm.batch.job; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.batch.reader.BaseReverseCronologicalBatchPidReader; -import ca.uhn.fhir.jpa.dao.data.IMdmLinkDao; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * This is the same as the parent class, except it operates on MdmLink entities instead of resource entities - */ -public class ReverseCronologicalBatchMdmLinkPidReader extends BaseReverseCronologicalBatchPidReader { - @Autowired - IMdmLinkDao myMdmLinkDao; - - @Override - protected Set getNextPidBatch(ResourceSearch resourceSearch) { - String resourceName = resourceSearch.getResourceName(); - Pageable pageable = PageRequest.of(0, getBatchSize()); - RequestPartitionId requestPartitionId = resourceSearch.getRequestPartitionId(); - if (requestPartitionId.isAllPartitions()){ - return new HashSet<>(myMdmLinkDao.findPidByResourceNameAndThreshold(resourceName, getCurrentHighThreshold(), pageable)); - } - List partitionIds = requestPartitionId.getPartitionIds(); - //Expand out the list to handle the REDIRECT/POSSIBLE DUPLICATE ones. - return new HashSet<>(myMdmLinkDao.findPidByResourceNameAndThresholdAndPartitionId(resourceName, getCurrentHighThreshold(), partitionIds, pageable)); - } - - @Override - protected void setDateFromPidFunction(ResourceSearch resourceSearch) { - setDateExtractorFunction(pid -> myMdmLinkDao.findById(pid).get().getCreated()); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java deleted file mode 100644 index 0278500dae3..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BaseReverseCronologicalBatchPidReader.java +++ /dev/null @@ -1,240 +0,0 @@ -package ca.uhn.fhir.jpa.batch.reader; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.CommonBatchJobConfig; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; -import ca.uhn.fhir.jpa.batch.job.model.PartitionedUrl; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.api.Constants; -import ca.uhn.fhir.rest.api.SortOrderEnum; -import ca.uhn.fhir.rest.api.SortSpec; -import ca.uhn.fhir.rest.param.DateRangeParam; -import ca.uhn.fhir.rest.param.ParamPrefixEnum; -import org.apache.commons.lang3.time.DateUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.core.JobParameter; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemStream; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; - -import javax.annotation.Nonnull; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -/** - * This Spring Batch reader takes 4 parameters: - * {@link BatchConstants#JOB_PARAM_REQUEST_LIST}: A list of URLs to search for along with the partitions those searches should be performed on - * {@link BatchConstants#JOB_PARAM_BATCH_SIZE}: The number of resources to return with each search. If ommitted, {@link DaoConfig#getExpungeBatchSize} will be used. - * {@link BatchConstants#JOB_PARAM_START_TIME}: The latest timestamp of entities to search for - *

- * The reader will return at most {@link BatchConstants#JOB_PARAM_BATCH_SIZE} pids every time it is called, or null - * once no more matching entities are available. It returns the resources in reverse chronological order - * and stores where it's at in the Spring Batch execution context with the key {@link BatchConstants#CURRENT_THRESHOLD_HIGH} - * appended with "." and the index number of the url list item it has gotten up to. This is to permit - * restarting jobs that use this reader so it can pick up where it left off. - */ -public abstract class BaseReverseCronologicalBatchPidReader implements ItemReader>, ItemStream { - private static final Logger ourLog = LoggerFactory.getLogger(ReverseCronologicalBatchResourcePidReader.class); - private final BatchDateThresholdUpdater myBatchDateThresholdUpdater = new BatchDateThresholdUpdater(); - private final Map myThresholdHighByUrlIndex = new HashMap<>(); - private final Map> myAlreadyProcessedPidsWithHighDate = new HashMap<>(); - @Autowired - private FhirContext myFhirContext; - @Autowired - private MatchUrlService myMatchUrlService; - private List myPartitionedUrls; - private Integer myBatchSize; - private int myUrlIndex = 0; - private Date myStartTime; - - private static String highKey(int theIndex) { - return BatchConstants.CURRENT_THRESHOLD_HIGH + "." + theIndex; - } - - @Nonnull - public static JobParameters buildJobParameters(String theOperationName, Integer theBatchSize, RequestListJson theRequestListJson) { - Map map = new HashMap<>(); - map.put(MultiUrlJobParameterValidator.JOB_PARAM_OPERATION_NAME, new JobParameter(theOperationName)); - map.put(BatchConstants.JOB_PARAM_REQUEST_LIST, new JobParameter(theRequestListJson.toJson())); - map.put(BatchConstants.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), CommonBatchJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); - if (theBatchSize != null) { - map.put(BatchConstants.JOB_PARAM_BATCH_SIZE, new JobParameter(theBatchSize.longValue())); - } - JobParameters parameters = new JobParameters(map); - return parameters; - } - - @Autowired - public void setRequestListJson(@Value("#{jobParameters['" + BatchConstants.JOB_PARAM_REQUEST_LIST + "']}") String theRequestListJson) { - RequestListJson requestListJson = RequestListJson.fromJson(theRequestListJson); - myPartitionedUrls = requestListJson.getPartitionedUrls(); - } - - @Autowired - public void setStartTime(@Value("#{jobParameters['" + BatchConstants.JOB_PARAM_START_TIME + "']}") Date theStartTime) { - myStartTime = theStartTime; - } - - @Override - public List read() throws Exception { - while (myUrlIndex < myPartitionedUrls.size()) { - List nextBatch = getNextBatch(); - if (nextBatch.isEmpty()) { - ++myUrlIndex; - continue; - } - - return nextBatch; - } - return null; - } - - protected List getNextBatch() { - RequestPartitionId requestPartitionId = myPartitionedUrls.get(myUrlIndex).getRequestPartitionId(); - ResourceSearch resourceSearch = myMatchUrlService.getResourceSearch(myPartitionedUrls.get(myUrlIndex).getUrl(), requestPartitionId); - myAlreadyProcessedPidsWithHighDate.putIfAbsent(myUrlIndex, new HashSet<>()); - Set newPids = getNextPidBatch(resourceSearch); - - if (ourLog.isDebugEnabled()) { - ourLog.debug("Search for {}{} returned {} results", resourceSearch.getResourceName(), resourceSearch.getSearchParameterMap().toNormalizedQueryString(myFhirContext), newPids.size()); - ourLog.debug("Results: {}", newPids); - } - - setDateFromPidFunction(resourceSearch); - - List retval = new ArrayList<>(newPids); - Date newThreshold = myBatchDateThresholdUpdater.updateThresholdAndCache(getCurrentHighThreshold(), myAlreadyProcessedPidsWithHighDate.get(myUrlIndex), retval); - myThresholdHighByUrlIndex.put(myUrlIndex, newThreshold); - - return retval; - } - - protected Date getCurrentHighThreshold() { - return myThresholdHighByUrlIndex.get(myUrlIndex); - } - - protected void setDateExtractorFunction(Function theDateExtractorFunction) { - myBatchDateThresholdUpdater.setDateFromPid(theDateExtractorFunction); - } - - protected void addDateCountAndSortToSearch(ResourceSearch resourceSearch) { - SearchParameterMap map = resourceSearch.getSearchParameterMap(); - DateRangeParam rangeParam = getDateRangeParam(resourceSearch); - map.setLastUpdated(rangeParam); - map.setLoadSynchronousUpTo(myBatchSize); - map.setSort(new SortSpec(Constants.PARAM_LASTUPDATED, SortOrderEnum.DESC)); - } - - /** - * Evaluates the passed in {@link ResourceSearch} to see if it contains a non-null {@link DateRangeParam}. - * - * If one such {@link DateRangeParam} exists, we use that to determine the upper and lower bounds for the returned - * {@link DateRangeParam}. The {@link DateRangeParam#getUpperBound()} is compared to the - * {@link BaseReverseCronologicalBatchPidReader#getCurrentHighThreshold()}, and the lower of the two date values - * is used. - * - * If no {@link DateRangeParam} is set, we use the local {@link BaseReverseCronologicalBatchPidReader#getCurrentHighThreshold()} - * to create a {@link DateRangeParam}. - * @param resourceSearch The {@link ResourceSearch} to check. - * @return {@link DateRangeParam} - */ - private DateRangeParam getDateRangeParam(ResourceSearch resourceSearch) { - DateRangeParam rangeParam = resourceSearch.getSearchParameterMap().getLastUpdated(); - if (rangeParam != null) { - if (rangeParam.getUpperBound() == null) { - rangeParam.setUpperBoundInclusive(getCurrentHighThreshold()); - } else { - Date theUpperBound = (getCurrentHighThreshold() == null || rangeParam.getUpperBound().getValue().before(getCurrentHighThreshold())) - ? rangeParam.getUpperBound().getValue() : getCurrentHighThreshold(); - rangeParam.setUpperBoundInclusive(theUpperBound); - } - } else { - rangeParam = new DateRangeParam().setUpperBoundInclusive(getCurrentHighThreshold()); - } - return rangeParam; - } - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - if (executionContext.containsKey(BatchConstants.CURRENT_URL_INDEX)) { - myUrlIndex = new Long(executionContext.getLong(BatchConstants.CURRENT_URL_INDEX)).intValue(); - } - for (int index = 0; index < myPartitionedUrls.size(); ++index) { - String key = highKey(index); - if (executionContext.containsKey(key)) { - myThresholdHighByUrlIndex.put(index, new Date(executionContext.getLong(key))); - } else { - myThresholdHighByUrlIndex.put(index, myStartTime); - } - } - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - executionContext.putLong(BatchConstants.CURRENT_URL_INDEX, myUrlIndex); - for (int index = 0; index < myPartitionedUrls.size(); ++index) { - Date date = myThresholdHighByUrlIndex.get(index); - if (date != null) { - executionContext.putLong(highKey(index), date.getTime()); - } - } - } - - @Override - public void close() throws ItemStreamException { - } - - protected Integer getBatchSize() { - return myBatchSize; - } - - @Autowired - public void setBatchSize(@Value("#{jobParameters['" + BatchConstants.JOB_PARAM_BATCH_SIZE + "']}") Integer theBatchSize) { - myBatchSize = theBatchSize; - } - - protected Set getAlreadySeenPids() { - return myAlreadyProcessedPidsWithHighDate.get(myUrlIndex); - } - - protected abstract Set getNextPidBatch(ResourceSearch resourceSearch); - - protected abstract void setDateFromPidFunction(ResourceSearch resourceSearch); -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdater.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdater.java deleted file mode 100644 index 42151e61bc6..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdater.java +++ /dev/null @@ -1,96 +0,0 @@ -package ca.uhn.fhir.jpa.batch.reader; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Date; -import java.util.List; -import java.util.Set; -import java.util.function.Function; - -public class BatchDateThresholdUpdater { - private static final Logger ourLog = LoggerFactory.getLogger(BatchDateThresholdUpdater.class); - - private Function myDateFromPid; - - public BatchDateThresholdUpdater() { - } - - public BatchDateThresholdUpdater(Function theDateFromPid) { - myDateFromPid = theDateFromPid; - } - - /** - * This method is used by batch jobs that process resource pids by date in multiple passes. It's used to ensure - * the same resource isn't processed twice. What it does is after a pass of processing pids, it sets - * the threshold date for the next pass from the last resource on the list and collects all of the resources that have that date into a temporary cache - * so that the caller can exclude those from the next pass. - * - * @param thePrevThreshold the date threshold from the previous pass - * @param theAlreadyProcessedPidsWithThresholdDate the set to load pids into that have the new threshold - * @param theProcessedPidsOrderedByDate the pids ordered by date (can be ascending or descending) - * @return the new date threshold (can be the same as the old threshold if all pids on the list share the same date) - */ - - public Date updateThresholdAndCache(Date thePrevThreshold, Set theAlreadyProcessedPidsWithThresholdDate, List theProcessedPidsOrderedByDate) { - if (theProcessedPidsOrderedByDate.isEmpty()) { - return thePrevThreshold; - } - - // Adjust the low threshold to be the last resource in the batch we found - Long pidOfLatestResourceInBatch = theProcessedPidsOrderedByDate.get(theProcessedPidsOrderedByDate.size() - 1); - Date latestUpdatedDate = myDateFromPid.apply(pidOfLatestResourceInBatch); - - // The latest date has changed, create a new cache to store pids with that date - if (thePrevThreshold != latestUpdatedDate) { - theAlreadyProcessedPidsWithThresholdDate.clear(); - } - theAlreadyProcessedPidsWithThresholdDate.add(pidOfLatestResourceInBatch); - - Date newThreshold = latestUpdatedDate; - if (theProcessedPidsOrderedByDate.size() <= 1) { - return newThreshold; - } - - // There is more than one resource in this batch, add any others with the same date. Assume the list is ordered by date. - for (int index = theProcessedPidsOrderedByDate.size() - 2; index >= 0; --index) { - Long pid = theProcessedPidsOrderedByDate.get(index); - Date newDate = myDateFromPid.apply(pid); - if (!latestUpdatedDate.equals(newDate)) { - break; - } - theAlreadyProcessedPidsWithThresholdDate.add(pid); - } - - return newThreshold; - } - - /** - * @param theDateFromPid this is a Function to extract a date from a resource id - * @return - */ - public BatchDateThresholdUpdater setDateFromPid(Function theDateFromPid) { - myDateFromPid = theDateFromPid; - return this; - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchResourceSearcher.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchResourceSearcher.java deleted file mode 100644 index f907173b36e..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/BatchResourceSearcher.java +++ /dev/null @@ -1,65 +0,0 @@ -package ca.uhn.fhir.jpa.batch.reader; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; -import ca.uhn.fhir.jpa.dao.IResultIterator; -import ca.uhn.fhir.jpa.dao.ISearchBuilder; -import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; -import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; -import ca.uhn.fhir.jpa.partition.SystemRequestDetails; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.annotation.Nonnull; -import java.util.UUID; - -/** - * This service is used by batch processes to search resources - */ -public class BatchResourceSearcher { - @Autowired - private SearchBuilderFactory mySearchBuilderFactory; - @Autowired - private DaoRegistry myDaoRegistry; - - public IResultIterator performSearch(ResourceSearch theResourceSearch, Integer theBatchSize) { - String resourceName = theResourceSearch.getResourceName(); - RequestPartitionId requestPartitionId = theResourceSearch.getRequestPartitionId(); - - IFhirResourceDao dao = myDaoRegistry.getResourceDao(resourceName); - final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(dao, resourceName, theResourceSearch.getResourceType()); - sb.setFetchSize(theBatchSize); - SystemRequestDetails requestDetails = buildSystemRequestDetails(requestPartitionId); - SearchRuntimeDetails searchRuntimeDetails = new SearchRuntimeDetails(requestDetails, UUID.randomUUID().toString()); - IResultIterator resultIter = sb.createQuery(theResourceSearch.getSearchParameterMap(), searchRuntimeDetails, requestDetails, requestPartitionId); - return resultIter; - } - - @Nonnull - private SystemRequestDetails buildSystemRequestDetails(RequestPartitionId theRequestPartitionId) { - SystemRequestDetails retval = new SystemRequestDetails(); - retval.setRequestPartitionId(theRequestPartitionId); - return retval; - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java deleted file mode 100644 index f96061c9b13..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/CronologicalBatchAllResourcePidReader.java +++ /dev/null @@ -1,163 +0,0 @@ -package ca.uhn.fhir.jpa.batch.reader; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.CommonBatchJobConfig; -import ca.uhn.fhir.jpa.dao.data.IResourceTableDao; -import ca.uhn.fhir.jpa.model.entity.ResourceTable; -import com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.commons.lang3.time.DateUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.core.JobParameter; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemStream; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Slice; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * This Spring Batch reader takes 3 parameters: - * {@link #JOB_PARAM_BATCH_SIZE}: The number of resources to return with each search. - * {@link #JOB_PARAM_START_TIME}: The latest timestamp of resources to search for - * {@link #JOB_PARAM_REQUEST_PARTITION}: (optional) The partition of resources to read - *

- * The reader will return at most {@link #JOB_PARAM_BATCH_SIZE} pids every time it is called, or null - * once no more matching resources are available. It returns the resources in reverse chronological order - * appended with "." and the index number of the url list item it has gotten up to. This is to permit - * restarting jobs that use this reader so it can pick up where it left off. - */ -public class CronologicalBatchAllResourcePidReader implements ItemReader>, ItemStream { - public static final String JOB_PARAM_BATCH_SIZE = "batch-size"; - public static final String JOB_PARAM_START_TIME = "start-time"; - public static final String JOB_PARAM_REQUEST_PARTITION = "request-partition"; - public static final String CURRENT_THRESHOLD_LOW = "current.threshold-low"; - - private static final Logger ourLog = LoggerFactory.getLogger(CronologicalBatchAllResourcePidReader.class); - private static final Date BEGINNING_OF_TIME = new Date(0); - - @Autowired - private IResourceTableDao myResourceTableDao; - @Autowired - private DaoConfig myDaoConfig; - - private Integer myBatchSize; - private Date myThresholdLow = BEGINNING_OF_TIME; - private final BatchDateThresholdUpdater myBatchDateThresholdUpdater = new BatchDateThresholdUpdater(this::dateFromPid); - private final Set myAlreadyProcessedPidsWithLowDate = new HashSet<>(); - private Date myStartTime; - private RequestPartitionId myRequestPartitionId; - - @Autowired - public void setBatchSize(@Value("#{jobParameters['" + JOB_PARAM_BATCH_SIZE + "']}") Integer theBatchSize) { - myBatchSize = theBatchSize; - } - - @Autowired - public void setStartTime(@Value("#{jobParameters['" + JOB_PARAM_START_TIME + "']}") Date theStartTime) { - myStartTime = theStartTime; - } - - public static JobParameters buildJobParameters(Integer theBatchSize, RequestPartitionId theRequestPartitionId) { - Map map = new HashMap<>(); - map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_REQUEST_PARTITION, new JobParameter(theRequestPartitionId.toJson())); - map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_START_TIME, new JobParameter(DateUtils.addMinutes(new Date(), CommonBatchJobConfig.MINUTES_IN_FUTURE_TO_PROCESS_FROM))); - if (theBatchSize != null) { - map.put(CronologicalBatchAllResourcePidReader.JOB_PARAM_BATCH_SIZE, new JobParameter(theBatchSize.longValue())); - } - JobParameters parameters = new JobParameters(map); - return parameters; - } - - @Override - public List read() throws Exception { - List nextBatch = getNextBatch(); - return nextBatch.isEmpty() ? null : nextBatch; - } - - private Date dateFromPid(Long thePid) { - ResourceTable entity = myResourceTableDao.findById(thePid).orElseThrow(IllegalStateException::new); - return entity.getUpdatedDate(); - } - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - if (myBatchSize == null) { - myBatchSize = myDaoConfig.getExpungeBatchSize(); - } - if (executionContext.containsKey(CURRENT_THRESHOLD_LOW)) { - myThresholdLow = new Date(executionContext.getLong(CURRENT_THRESHOLD_LOW)); - } - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - executionContext.putLong(CURRENT_THRESHOLD_LOW, myThresholdLow.getTime()); - } - - @Override - public void close() throws ItemStreamException { - } - - private List getNextBatch() { - PageRequest page = PageRequest.of(0, myBatchSize); - List retval = new ArrayList<>(); - Slice slice; - do { - if (myRequestPartitionId == null || myRequestPartitionId.isAllPartitions()) { - slice = myResourceTableDao.findIdsOfResourcesWithinUpdatedRangeOrderedFromOldest(page, myThresholdLow, myStartTime); - } else { - slice = myResourceTableDao.findIdsOfPartitionedResourcesWithinUpdatedRangeOrderedFromOldest(page, myThresholdLow, myStartTime, myRequestPartitionId.getFirstPartitionIdOrNull()); - } - retval.addAll(slice.getContent()); - retval.removeAll(myAlreadyProcessedPidsWithLowDate); - page = page.next(); - } while (retval.size() < myBatchSize && slice.hasNext()); - - if (ourLog.isDebugEnabled()) { - ourLog.debug("Results: {}", retval); - } - myThresholdLow = myBatchDateThresholdUpdater.updateThresholdAndCache(myThresholdLow, myAlreadyProcessedPidsWithLowDate, retval); - return retval; - } - - @Autowired - public void setRequestPartitionId(@Value("#{jobParameters['" + JOB_PARAM_REQUEST_PARTITION + "']}") String theRequestPartitionIdJson) throws JsonProcessingException { - if (theRequestPartitionIdJson == null) { - return; - } - myRequestPartitionId = RequestPartitionId.fromJson(theRequestPartitionIdJson); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java deleted file mode 100644 index e7b33ced988..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReader.java +++ /dev/null @@ -1,71 +0,0 @@ -package ca.uhn.fhir.jpa.batch.reader; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; -import ca.uhn.fhir.jpa.dao.IResultIterator; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; -import org.hl7.fhir.instance.model.api.IBaseResource; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - - -public class ReverseCronologicalBatchResourcePidReader extends BaseReverseCronologicalBatchPidReader { - @Autowired - private DaoRegistry myDaoRegistry; - @Autowired - private BatchResourceSearcher myBatchResourceSearcher; - - @Override - protected Set getNextPidBatch(ResourceSearch resourceSearch) { - Set retval = new LinkedHashSet<>(); - addDateCountAndSortToSearch(resourceSearch); - - // Perform the search - Integer batchSize = getBatchSize(); - IResultIterator resultIter = myBatchResourceSearcher.performSearch(resourceSearch, batchSize); - Set alreadySeenPids = getAlreadySeenPids(); - - do { - List pids = resultIter.getNextResultBatch(batchSize).stream().map(ResourcePersistentId::getIdAsLong).collect(Collectors.toList()); - retval.addAll(pids); - retval.removeAll(alreadySeenPids); - } while (retval.size() < batchSize && resultIter.hasNext()); - - return retval; - } - - @Override - protected void setDateFromPidFunction(ResourceSearch resourceSearch) { - final IFhirResourceDao dao = myDaoRegistry.getResourceDao(resourceSearch.getResourceName()); - - setDateExtractorFunction(pid -> { - IBaseResource oldestResource = dao.readByPid(new ResourcePersistentId(pid)); - return oldestResource.getMeta().getLastUpdated(); - }); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/writer/SqlExecutorWriter.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/writer/SqlExecutorWriter.java deleted file mode 100644 index 8df29b55d85..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch/writer/SqlExecutorWriter.java +++ /dev/null @@ -1,69 +0,0 @@ -package ca.uhn.fhir.jpa.batch.writer; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.core.StepExecution; -import org.springframework.batch.core.annotation.BeforeStep; -import org.springframework.batch.item.ItemWriter; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.PersistenceContextType; -import java.util.List; - -/** - * This Spring Batch writer accepts a list of SQL commands and executes them. - * The total number of entities updated or deleted is stored in the execution context - * with the key {@link #ENTITY_TOTAL_UPDATED_OR_DELETED}. The entire list is committed within a - * single transaction (provided by Spring Batch). - */ -public class SqlExecutorWriter implements ItemWriter> { - private static final Logger ourLog = LoggerFactory.getLogger(SqlExecutorWriter.class); - - public static final String ENTITY_TOTAL_UPDATED_OR_DELETED = "entity.total.updated-or-deleted"; - - @PersistenceContext(type = PersistenceContextType.TRANSACTION) - private EntityManager myEntityManager; - private Long totalUpdated = 0L; - private StepExecution myStepExecution; - - @BeforeStep - public void setStepExecution(StepExecution stepExecution) { - myStepExecution = stepExecution; - } - - @Override - public void write(List> theSqlLists) throws Exception { - - // Note that since our chunk size is 1, there will always be exactly one list - for (List sqlList : theSqlLists) { - ourLog.info("Executing {} sql commands", sqlList.size()); - for (String sql : sqlList) { - ourLog.trace("Executing sql " + sql); - totalUpdated += myEntityManager.createNativeQuery(sql).executeUpdate(); - myStepExecution.getExecutionContext().putLong(ENTITY_TOTAL_UPDATED_OR_DELETED, totalUpdated); - } - } - ourLog.debug("{} records updated", totalUpdated); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/Batch2SupportConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/Batch2SupportConfig.java new file mode 100644 index 00000000000..4a20654b884 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/Batch2SupportConfig.java @@ -0,0 +1,52 @@ +package ca.uhn.fhir.jpa.config; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.api.config.DaoConfig; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; +import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao; +import ca.uhn.fhir.jpa.dao.expunge.ResourceTableFKProvider; +import ca.uhn.fhir.jpa.dao.index.IJpaIdHelperService; +import ca.uhn.fhir.jpa.delete.batch2.DeleteExpungeSqlBuilder; +import ca.uhn.fhir.jpa.delete.batch2.DeleteExpungeSvcImpl; +import ca.uhn.fhir.jpa.reindex.Batch2DaoSvcImpl; +import org.springframework.context.annotation.Bean; + +import javax.persistence.EntityManager; + +public class Batch2SupportConfig { + + @Bean + public IBatch2DaoSvc batch2DaoSvc() { + return new Batch2DaoSvcImpl(); + } + + @Bean + public IDeleteExpungeSvc deleteExpungeSvc(EntityManager theEntityManager, DeleteExpungeSqlBuilder theDeleteExpungeSqlBuilder) { + return new DeleteExpungeSvcImpl(theEntityManager, theDeleteExpungeSqlBuilder); + } + + @Bean + DeleteExpungeSqlBuilder deleteExpungeSqlBuilder(ResourceTableFKProvider theResourceTableFKProvider, DaoConfig theDaoConfig, IJpaIdHelperService theIdHelper, IResourceLinkDao theResourceLinkDao) { + return new DeleteExpungeSqlBuilder(theResourceTableFKProvider, theDaoConfig, theIdHelper, theResourceLinkDao); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java index 02fb03a691a..6501088ba5a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java @@ -1,5 +1,6 @@ package ca.uhn.fhir.jpa.config; +import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeJobSubmitterImpl; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.IValidationSupport; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; @@ -9,13 +10,9 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IDao; import ca.uhn.fhir.jpa.api.model.ExpungeOptions; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.batch.BatchJobsConfig; import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.batch.job.PartitionedUrlValidator; -import ca.uhn.fhir.jpa.batch.mdm.batch.MdmClearJobSubmitterImpl; -import ca.uhn.fhir.jpa.batch.reader.BatchResourceSearcher; import ca.uhn.fhir.jpa.binary.interceptor.BinaryStorageInterceptor; import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider; import ca.uhn.fhir.jpa.bulk.export.api.IBulkDataExportJobSchedulingHelper; @@ -64,7 +61,6 @@ import ca.uhn.fhir.jpa.dao.predicate.PredicateBuilderUri; import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; import ca.uhn.fhir.jpa.delete.DeleteConflictFinderService; import ca.uhn.fhir.jpa.delete.DeleteConflictService; -import ca.uhn.fhir.jpa.delete.DeleteExpungeJobSubmitterImpl; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.graphql.DaoRegistryGraphQLStorageServices; import ca.uhn.fhir.jpa.interceptor.CascadingDeleteInterceptor; @@ -87,15 +83,14 @@ import ca.uhn.fhir.jpa.provider.SubscriptionTriggeringProvider; import ca.uhn.fhir.jpa.provider.TerminologyUploaderProvider; import ca.uhn.fhir.jpa.provider.ValueSetOperationProvider; import ca.uhn.fhir.jpa.provider.r4.MemberMatcherR4Helper; -import ca.uhn.fhir.jpa.reindex.ResourceReindexSvcImpl; import ca.uhn.fhir.jpa.sched.AutowiringSpringBeanJobFactory; import ca.uhn.fhir.jpa.sched.HapiSchedulerServiceImpl; -import ca.uhn.fhir.jpa.search.SearchStrategyFactory; import ca.uhn.fhir.jpa.search.ISynchronousSearchSvc; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider; import ca.uhn.fhir.jpa.search.PersistedJpaBundleProviderFactory; import ca.uhn.fhir.jpa.search.PersistedJpaSearchFirstPageBundleProvider; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; +import ca.uhn.fhir.jpa.search.SearchStrategyFactory; import ca.uhn.fhir.jpa.search.SynchronousSearchSvcImpl; import ca.uhn.fhir.jpa.search.builder.QueryStack; import ca.uhn.fhir.jpa.search.builder.SearchBuilder; @@ -141,14 +136,12 @@ import ca.uhn.fhir.jpa.term.api.ITermConceptMappingSvc; import ca.uhn.fhir.jpa.util.MemoryCacheService; import ca.uhn.fhir.jpa.validation.JpaResourceLoader; import ca.uhn.fhir.jpa.validation.ValidationSettings; -import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.server.interceptor.ResponseTerminologyTranslationInterceptor; import ca.uhn.fhir.rest.server.interceptor.consent.IConsentContextServices; import ca.uhn.fhir.rest.server.interceptor.partition.RequestTenantPartitionInterceptor; -import ca.uhn.fhir.rest.server.provider.DeleteExpungeProvider; import ca.uhn.fhir.util.ThreadPoolUtil; import org.hl7.fhir.common.hapi.validation.support.UnknownCodeSystemWarningValidationSupport; import org.hl7.fhir.instance.model.api.IBaseResource; @@ -199,7 +192,8 @@ import java.util.Date; BeanPostProcessorConfig.class, BatchJobsConfig.class, SearchParamConfig.class, - ValidationSupportConfig.class + ValidationSupportConfig.class, + Batch2SupportConfig.class }) public class JpaConfig { public static final String JPA_VALIDATION_SUPPORT_CHAIN = "myJpaValidationSupportChain"; @@ -355,11 +349,6 @@ public class JpaConfig { return new ResourceReindexer(theFhirContext); } - @Bean - public BatchResourceSearcher myBatchResourceSearcher() { - return new BatchResourceSearcher(); - } - @Bean public HapiFhirHibernateJpaDialect hibernateJpaDialect(FhirContext theFhirContext) { return new HapiFhirHibernateJpaDialect(theFhirContext.getLocalizer()); @@ -426,12 +415,6 @@ public class JpaConfig { return new MdmLinkExpandSvc(); } - - @Bean - IMdmClearJobSubmitter mdmClearJobSubmitter() { - return new MdmClearJobSubmitterImpl(); - } - @Bean @Lazy public TerminologyUploaderProvider terminologyUploaderProvider() { @@ -471,18 +454,6 @@ public class JpaConfig { return new DeleteExpungeJobSubmitterImpl(); } - @Bean - @Lazy - public PartitionedUrlValidator partitionedUrlValidator() { - return new PartitionedUrlValidator(); - } - - @Bean - @Lazy - public DeleteExpungeProvider deleteExpungeProvider(FhirContext theFhirContext, IDeleteExpungeJobSubmitter theDeleteExpungeJobSubmitter) { - return new DeleteExpungeProvider(theFhirContext, theDeleteExpungeJobSubmitter); - } - @Bean @Lazy public IBulkDataImportSvc bulkDataImportSvc() { @@ -519,11 +490,6 @@ public class JpaConfig { return new ResourceVersionSvcDaoImpl(); } - @Bean - public IResourceReindexSvc resourceReindexSvc() { - return new ResourceReindexSvcImpl(); - } - /* **************************************************************** * * Prototype Beans Below * * **************************************************************** */ diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java index 6d97334bfc0..1303e405e93 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java @@ -117,8 +117,6 @@ import org.hl7.fhir.instance.model.api.IBaseParameters; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Required; import org.springframework.transaction.PlatformTransactionManager; @@ -670,9 +668,9 @@ public abstract class BaseHapiFhirResourceDao extends B List urlsToDeleteExpunge = Collections.singletonList(theUrl); try { - JobExecution jobExecution = myDeleteExpungeJobSubmitter.submitJob(getConfig().getExpungeBatchSize(), urlsToDeleteExpunge, theRequest); - return new DeleteMethodOutcome(createInfoOperationOutcome("Delete job submitted with id " + jobExecution.getId())); - } catch (JobParametersInvalidException e) { + String jobId = myDeleteExpungeJobSubmitter.submitJob(getConfig().getExpungeBatchSize(), urlsToDeleteExpunge, theRequest); + return new DeleteMethodOutcome(createInfoOperationOutcome("Delete job submitted with id " + jobId)); + } catch (InvalidRequestException e) { throw new InvalidRequestException(Msg.code(965) + "Invalid Delete Expunge Request: " + e.getMessage(), e); } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeProcessor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSqlBuilder.java similarity index 76% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeProcessor.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSqlBuilder.java index 11131fe1e9b..0ef136980ea 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeProcessor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSqlBuilder.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.jpa.delete.job; +package ca.uhn.fhir.jpa.delete.batch2; /*- * #%L @@ -23,7 +23,6 @@ package ca.uhn.fhir.jpa.delete.job; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao; -import ca.uhn.fhir.jpa.dao.expunge.PartitionRunner; import ca.uhn.fhir.jpa.dao.expunge.ResourceForeignKey; import ca.uhn.fhir.jpa.dao.expunge.ResourceTableFKProvider; import ca.uhn.fhir.jpa.dao.index.IJpaIdHelperService; @@ -32,51 +31,50 @@ import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.factory.annotation.Autowired; +import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.function.Consumer; import java.util.stream.Collectors; -/** - * Input: list of pids of resources to be deleted and expunged - * Output: list of sql statements to be executed - */ -public class DeleteExpungeProcessor implements ItemProcessor, List> { - private static final Logger ourLog = LoggerFactory.getLogger(DeleteExpungeProcessor.class); - +public class DeleteExpungeSqlBuilder { + private static final Logger ourLog = LoggerFactory.getLogger(DeleteExpungeSqlBuilder.class); public static final String PROCESS_NAME = "Delete Expunging"; public static final String THREAD_PREFIX = "delete-expunge"; + + private final ResourceTableFKProvider myResourceTableFKProvider; + private final DaoConfig myDaoConfig; + private final IJpaIdHelperService myIdHelper; + private final IResourceLinkDao myResourceLinkDao; - @Autowired - ResourceTableFKProvider myResourceTableFKProvider; - @Autowired - DaoConfig myDaoConfig; - @Autowired - IJpaIdHelperService myIdHelper; - @Autowired - IResourceLinkDao myResourceLinkDao; + public DeleteExpungeSqlBuilder(ResourceTableFKProvider theResourceTableFKProvider, DaoConfig theDaoConfig, IJpaIdHelperService theIdHelper, IResourceLinkDao theResourceLinkDao) { + myResourceTableFKProvider = theResourceTableFKProvider; + myDaoConfig = theDaoConfig; + myIdHelper = theIdHelper; + myResourceLinkDao = theResourceLinkDao; + } - @Override - public List process(List thePids) throws Exception { - validateOkToDeleteAndExpunge(thePids); - List retval = new ArrayList<>(); + @Nonnull + List convertPidsToDeleteExpungeSql(List thePersistentIds) { + List pids = ResourcePersistentId.toLongList(thePersistentIds); - String pidListString = thePids.toString().replace("[", "(").replace("]", ")"); + validateOkToDeleteAndExpunge(pids); + + List rawSql = new ArrayList<>(); + + String pidListString = pids.toString().replace("[", "(").replace("]", ")"); List resourceForeignKeys = myResourceTableFKProvider.getResourceForeignKeys(); for (ResourceForeignKey resourceForeignKey : resourceForeignKeys) { - retval.add(deleteRecordsByColumnSql(pidListString, resourceForeignKey)); + rawSql.add(deleteRecordsByColumnSql(pidListString, resourceForeignKey)); } // Lastly we need to delete records from the resource table all of these other tables link to: ResourceForeignKey resourceTablePk = new ResourceForeignKey("HFJ_RESOURCE", "RES_ID"); - retval.add(deleteRecordsByColumnSql(pidListString, resourceTablePk)); - return retval; + rawSql.add(deleteRecordsByColumnSql(pidListString, resourceTablePk)); + return rawSql; } public void validateOkToDeleteAndExpunge(List thePids) { @@ -87,9 +85,7 @@ public class DeleteExpungeProcessor implements ItemProcessor, List targetPidsAsResourceIds = ResourcePersistentId.fromLongList(thePids); List conflictResourceLinks = Collections.synchronizedList(new ArrayList<>()); - PartitionRunner partitionRunner = new PartitionRunner(PROCESS_NAME, THREAD_PREFIX, myDaoConfig.getExpungeBatchSize(), myDaoConfig.getExpungeThreadCount()); - Consumer> listConsumer = someTargetPids -> findResourceLinksWithTargetPidIn(targetPidsAsResourceIds, someTargetPids, conflictResourceLinks); - partitionRunner.runInPartitionedThreads(targetPidsAsResourceIds, listConsumer); + findResourceLinksWithTargetPidIn(targetPidsAsResourceIds, targetPidsAsResourceIds, conflictResourceLinks); if (conflictResourceLinks.isEmpty()) { return; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java new file mode 100644 index 00000000000..8d76de6460b --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/batch2/DeleteExpungeSvcImpl.java @@ -0,0 +1,60 @@ +package ca.uhn.fhir.jpa.delete.batch2; + +/*- + * #%L + * HAPI FHIR JPA Server + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.persistence.EntityManager; +import java.util.List; + +@Transactional(propagation = Propagation.MANDATORY) +public class DeleteExpungeSvcImpl implements IDeleteExpungeSvc { + private static final Logger ourLog = LoggerFactory.getLogger(DeleteExpungeSvcImpl.class); + + private final EntityManager myEntityManager; + private final DeleteExpungeSqlBuilder myDeleteExpungeSqlBuilder; + + public DeleteExpungeSvcImpl(EntityManager theEntityManager, DeleteExpungeSqlBuilder theDeleteExpungeSqlBuilder) { + myEntityManager = theEntityManager; + myDeleteExpungeSqlBuilder = theDeleteExpungeSqlBuilder; + } + + @Override + public void deleteExpunge(List thePersistentIds) { + List sqlList = myDeleteExpungeSqlBuilder.convertPidsToDeleteExpungeSql(thePersistentIds); + + ourLog.debug("Executing {} delete expunge sql commands", sqlList.size()); + long totalDeleted = 0; + for (String sql : sqlList) { + ourLog.trace("Executing sql " + sql); + totalDeleted += myEntityManager.createNativeQuery(sql).executeUpdate(); + } + ourLog.info("{} records deleted", totalDeleted); + // TODO KHS instead of logging progress, produce result chunks that get aggregated into a delete expunge report + } + + +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java deleted file mode 100644 index 892e2b58f24..00000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobConfig.java +++ /dev/null @@ -1,102 +0,0 @@ -package ca.uhn.fhir.jpa.delete.job; - -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; -import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; -import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; -import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; -import org.springframework.batch.core.configuration.annotation.StepScope; -import org.springframework.batch.core.listener.ExecutionContextPromotionListener; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Lazy; - -import java.util.List; - -import static ca.uhn.fhir.jpa.batch.config.BatchConstants.DELETE_EXPUNGE_JOB_NAME; - -/** - * Spring batch Job configuration file. Contains all necessary plumbing to run a - * Delete Expunge job. - */ -@Configuration -public class DeleteExpungeJobConfig { - public static final String DELETE_EXPUNGE_URL_LIST_STEP_NAME = "delete-expunge-url-list-step"; - - @Autowired - private StepBuilderFactory myStepBuilderFactory; - @Autowired - private JobBuilderFactory myJobBuilderFactory; - - @Autowired - private MultiUrlJobParameterValidator myMultiUrlProcessorParameterValidator; - - @Autowired - private PidReaderCounterListener myPidCountRecorderListener; - - @Autowired - private ReverseCronologicalBatchResourcePidReader myReverseCronologicalBatchResourcePidReader; - - @Autowired - private SqlExecutorWriter mySqlExecutorWriter; - - @Bean(name = DELETE_EXPUNGE_JOB_NAME) - @Lazy - public Job deleteExpungeJob() { - return myJobBuilderFactory.get(DELETE_EXPUNGE_JOB_NAME) - .validator(myMultiUrlProcessorParameterValidator) - .start(deleteExpungeUrlListStep()) - .build(); - } - - @Bean - public Step deleteExpungeUrlListStep() { - return myStepBuilderFactory.get(DELETE_EXPUNGE_URL_LIST_STEP_NAME) - ., List>chunk(1) - .reader(myReverseCronologicalBatchResourcePidReader) - .processor(deleteExpungeProcessor()) - .writer(mySqlExecutorWriter) - .listener(myPidCountRecorderListener) - .listener(deleteExpungePromotionListener()) - .build(); - } - - @Bean - public ExecutionContextPromotionListener deleteExpungePromotionListener() { - ExecutionContextPromotionListener listener = new ExecutionContextPromotionListener(); - - listener.setKeys(new String[]{SqlExecutorWriter.ENTITY_TOTAL_UPDATED_OR_DELETED, PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED}); - - return listener; - } - - @Bean - @StepScope - public DeleteExpungeProcessor deleteExpungeProcessor() { - return new DeleteExpungeProcessor(); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/Batch2DaoSvcImpl.java similarity index 97% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImpl.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/Batch2DaoSvcImpl.java index 2bfdf3efdc4..91331428dd8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/reindex/Batch2DaoSvcImpl.java @@ -29,7 +29,7 @@ import ca.uhn.fhir.jpa.api.pid.EmptyResourcePidList; import ca.uhn.fhir.jpa.api.pid.HomogeneousResourcePidList; import ca.uhn.fhir.jpa.api.pid.IResourcePidList; import ca.uhn.fhir.jpa.api.pid.MixedResourcePidList; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; import ca.uhn.fhir.jpa.dao.data.IResourceTableDao; import ca.uhn.fhir.jpa.partition.SystemRequestDetails; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; @@ -51,7 +51,7 @@ import java.util.Date; import java.util.List; import java.util.stream.Collectors; -public class ResourceReindexSvcImpl implements IResourceReindexSvc { +public class Batch2DaoSvcImpl implements IBatch2DaoSvc { @Autowired private IResourceTableDao myResourceTableDao; diff --git a/hapi-fhir-jpaserver-cql/pom.xml b/hapi-fhir-jpaserver-cql/pom.xml index 91f9a04d4b0..fcc11136240 100644 --- a/hapi-fhir-jpaserver-cql/pom.xml +++ b/hapi-fhir-jpaserver-cql/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-mdm/pom.xml b/hapi-fhir-jpaserver-mdm/pom.xml index 9bdb6c6cbc4..e91ff6ed0e5 100644 --- a/hapi-fhir-jpaserver-mdm/pom.xml +++ b/hapi-fhir-jpaserver-mdm/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmCommonConfig.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmCommonConfig.java index 71e2cbed835..797c2f9f93a 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmCommonConfig.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/config/MdmCommonConfig.java @@ -21,10 +21,8 @@ package ca.uhn.fhir.jpa.mdm.config; */ import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.jpa.batch.mdm.batch.MdmBatchJobSubmitterFactoryImpl; import ca.uhn.fhir.jpa.dao.mdm.MdmLinkDeleteSvc; import ca.uhn.fhir.jpa.interceptor.MdmSearchExpandingInterceptor; -import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory; import ca.uhn.fhir.mdm.rules.config.MdmRuleValidator; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; import org.springframework.context.annotation.Bean; @@ -43,10 +41,6 @@ public class MdmCommonConfig { return new MdmSearchExpandingInterceptor(); } - @Bean - IMdmBatchJobSubmitterFactory mdmBatchJobSubmitterFactory() { - return new MdmBatchJobSubmitterFactoryImpl(); - } @Bean MdmLinkDeleteSvc mdmLinkDeleteSvc() { diff --git a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java index da998d8bebf..2f00d6e4831 100644 --- a/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java +++ b/hapi-fhir-jpaserver-mdm/src/main/java/ca/uhn/fhir/jpa/mdm/svc/MdmControllerSvcImpl.java @@ -27,7 +27,6 @@ import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; import ca.uhn.fhir.mdm.api.IGoldenResourceMergerSvc; -import ca.uhn.fhir.mdm.api.IMdmBatchJobSubmitterFactory; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; import ca.uhn.fhir.mdm.api.IMdmLinkCreateSvc; import ca.uhn.fhir.mdm.api.IMdmLinkQuerySvc; @@ -81,8 +80,6 @@ public class MdmControllerSvcImpl implements IMdmControllerSvc { @Autowired IMdmLinkCreateSvc myIMdmLinkCreateSvc; @Autowired - IMdmBatchJobSubmitterFactory myMdmBatchJobSubmitterFactory; - @Autowired IRequestPartitionHelperSvc myRequestPartitionHelperSvc; @Autowired IJobCoordinator myJobCoordinator; diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java index 43f3395b71b..2af442dc8d6 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/provider/BaseProviderR4Test.java @@ -4,7 +4,6 @@ import ca.uhn.fhir.batch2.api.IJobCoordinator; import ca.uhn.fhir.jpa.mdm.BaseMdmR4Test; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; import ca.uhn.fhir.jpa.test.Batch2JobHelper; -import ca.uhn.fhir.mdm.api.IMdmClearJobSubmitter; import ca.uhn.fhir.mdm.api.IMdmControllerSvc; import ca.uhn.fhir.mdm.api.IMdmSubmitSvc; import ca.uhn.fhir.mdm.provider.MdmControllerHelper; @@ -34,8 +33,6 @@ public abstract class BaseProviderR4Test extends BaseMdmR4Test { @Autowired private IMdmControllerSvc myMdmControllerSvc; @Autowired - private IMdmClearJobSubmitter myMdmClearJobSubmitter; - @Autowired private IMdmSubmitSvc myMdmSubmitSvc; @Autowired private MdmSettings myMdmSettings; diff --git a/hapi-fhir-jpaserver-model/pom.xml b/hapi-fhir-jpaserver-model/pom.xml index adee5916e39..2550034629f 100644 --- a/hapi-fhir-jpaserver-model/pom.xml +++ b/hapi-fhir-jpaserver-model/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-searchparam/pom.xml b/hapi-fhir-jpaserver-searchparam/pom.xml index 7720639b238..adaa5fb7699 100755 --- a/hapi-fhir-jpaserver-searchparam/pom.xml +++ b/hapi-fhir-jpaserver-searchparam/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-subscription/pom.xml b/hapi-fhir-jpaserver-subscription/pom.xml index 82ce2a02907..a3a56b39036 100644 --- a/hapi-fhir-jpaserver-subscription/pom.xml +++ b/hapi-fhir-jpaserver-subscription/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-utilities/pom.xml b/hapi-fhir-jpaserver-test-utilities/pom.xml index 960a0c77b71..8459538d0c5 100644 --- a/hapi-fhir-jpaserver-test-utilities/pom.xml +++ b/hapi-fhir-jpaserver-test-utilities/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java index a4ef79c25a0..f8f8d79c183 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java @@ -24,12 +24,17 @@ import ca.uhn.fhir.batch2.api.IJobCoordinator; import ca.uhn.fhir.batch2.api.IJobMaintenanceService; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; +import org.awaitility.core.ConditionTimeoutException; import org.hamcrest.Matchers; import org.springframework.beans.factory.annotation.Autowired; +import java.util.Collection; +import java.util.List; + import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; public class Batch2JobHelper { @@ -39,11 +44,12 @@ public class Batch2JobHelper { @Autowired private IJobCoordinator myJobCoordinator; - public void awaitJobCompletion(String theId) { + public JobInstance awaitJobCompletion(String theId) { await().until(() -> { myJobMaintenanceService.runMaintenancePass(); return myJobCoordinator.getInstance(theId).getStatus(); }, equalTo(StatusEnum.COMPLETED)); + return myJobCoordinator.getInstance(theId); } public void awaitSingleChunkJobCompletion(String theId) { @@ -79,4 +85,38 @@ public class Batch2JobHelper { public void awaitGatedStepId(String theExpectedGatedStepId, String theInstanceId) { await().until(() -> theExpectedGatedStepId.equals(myJobCoordinator.getInstance(theInstanceId).getCurrentGatedStepId())); } + + public long getCombinedRecordsProcessed(String theJobId) { + JobInstance job = myJobCoordinator.getInstance(theJobId); + return job.getCombinedRecordsProcessed(); + + } + + // TODO KHS I don't think this works yet + public void awaitAllCompletions(String theJobDefinitionId) { + List instances = myJobCoordinator.getInstancesbyJobDefinitionIdAndEndedStatus(theJobDefinitionId, false, 100, 0); + awaitJobCompletions(instances); + } + + protected void awaitJobCompletions(Collection theJobInstances) { + // This intermittently fails for unknown reasons, so I've added a bunch + // of extra junk here to improve what we output when it fails + for (JobInstance jobInstance : theJobInstances) { + try { + awaitJobCompletion(jobInstance.getInstanceId()); + } catch (ConditionTimeoutException e) { + StringBuilder msg = new StringBuilder(); + msg.append("Failed waiting for job to complete.\n"); + msg.append("Error: ").append(e).append("\n"); + msg.append("Statuses:"); + for (JobInstance instance : theJobInstances) { + msg.append("\n * Execution ") + .append(instance.getInstanceId()) + .append(" has status ") + .append(instance.getStatus()); + } + fail(msg.toString()); + } + } + } } diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterUtil.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterUtil.java deleted file mode 100644 index 018d5393805..00000000000 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/job/MultiUrlJobParameterUtil.java +++ /dev/null @@ -1,29 +0,0 @@ -package ca.uhn.fhir.jpa.batch.job; - -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.batch.job.model.PartitionedUrl; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; -import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; -import ca.uhn.fhir.rest.server.provider.ProviderConstants; -import org.springframework.batch.core.JobParameters; - -import javax.annotation.Nonnull; -import java.util.ArrayList; -import java.util.List; - -public final class MultiUrlJobParameterUtil { - private MultiUrlJobParameterUtil() { - } - - @Nonnull - public static JobParameters buildJobParameters(String... theUrls) { - List partitionedUrls = new ArrayList<>(); - for (String url : theUrls) { - partitionedUrls.add(new PartitionedUrl(url, RequestPartitionId.defaultPartition())); - } - - RequestListJson requestListJson = new RequestListJson(); - requestListJson.setPartitionedUrls(partitionedUrls); - return ReverseCronologicalBatchResourcePidReader.buildJobParameters(ProviderConstants.OPERATION_REINDEX, 2401, requestListJson); - } -} diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleterTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleterTest.java deleted file mode 100644 index c4102c83d22..00000000000 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/mdm/job/MdmLinkDeleterTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package ca.uhn.fhir.jpa.batch.mdm.job; - -import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.mdm.batch.job.MdmLinkDeleter; -import ca.uhn.fhir.jpa.dao.data.IMdmLinkDao; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.transaction.PlatformTransactionManager; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -public class MdmLinkDeleterTest { - @Mock - private DaoConfig myDaoConfig; - @Mock - private PlatformTransactionManager myPlatformTransactionManager; - @Mock - private IMdmLinkDao myMdmLinkDao; - - @Captor - ArgumentCaptor> myPidListCaptor; - - @InjectMocks - private MdmLinkDeleter myMdmLinkDeleter; - - @Test - public void testMdmLinkDeleterSplitsPidList() throws Exception { - int threadCount = 4; - int batchSize = 5; - when(myDaoConfig.getReindexBatchSize()).thenReturn(batchSize); - when(myDaoConfig.getReindexThreadCount()).thenReturn(threadCount); - - List allPidsList = new ArrayList<>(); - int count = threadCount*batchSize; - for (long i = 0; i < count; ++i) { - allPidsList.add(i); - } - myMdmLinkDeleter.process(allPidsList); - - verify(myMdmLinkDao, times(threadCount)).findAllById(myPidListCaptor.capture()); - verify(myMdmLinkDao, times(threadCount)).deleteAll(anyList()); - verifyNoMoreInteractions(myMdmLinkDao); - List> pidListList = myPidListCaptor.getAllValues(); - assertEquals(threadCount, pidListList.size()); - for (List pidList : pidListList) { - assertEquals(batchSize, pidList.size()); - } - } - -} diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdaterTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdaterTest.java deleted file mode 100644 index 36ef7692fd8..00000000000 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/BatchDateThresholdUpdaterTest.java +++ /dev/null @@ -1,125 +0,0 @@ -package ca.uhn.fhir.jpa.batch.reader; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.hasSize; -import static org.junit.jupiter.api.Assertions.assertEquals; - -@ExtendWith(MockitoExtension.class) -class BatchDateThresholdUpdaterTest { - static Date LATE_DATE = new Date(); - static Date EARLY_DATE = new Date(LATE_DATE.getTime() - 1000); - static Long PID1 = 1L; - static Long PID2 = 2L; - static Long PID3 = 3L; - BatchDateThresholdUpdater mySvc = new BatchDateThresholdUpdater(); - - @Test - public void testEmptyList() { - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, Collections.emptySet(), Collections.emptyList()); - assertEquals(LATE_DATE, newThreshold); - } - - @Test - public void oneItem() { - mySvc.setDateFromPid(pid -> LATE_DATE); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Collections.singletonList(PID1)); - assertEquals(LATE_DATE, newThreshold); - assertThat(seenPids, contains(PID1)); - } - - @Test - public void twoItemsSameDate() { - mySvc.setDateFromPid(pid -> LATE_DATE); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Arrays.asList(PID1, PID2)); - assertEquals(LATE_DATE, newThreshold); - assertThat(seenPids, contains(PID1, PID2)); - } - - @Test - public void twoItemsDiffDate() { - List dates = Arrays.asList(EARLY_DATE, LATE_DATE); - mySvc.setDateFromPid(pid -> dates.get(pid.intValue() - 1)); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Arrays.asList(PID1, PID2)); - assertEquals(LATE_DATE, newThreshold); - assertThat(seenPids, contains(PID2)); - } - - @Test - public void threeItemsSameDate() { - mySvc.setDateFromPid(pid -> LATE_DATE); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Arrays.asList(PID1, PID2, PID3)); - assertEquals(LATE_DATE, newThreshold); - assertThat(seenPids, contains(PID1, PID2, PID3)); - } - - @Test - public void threeItemsDifferentEEL() { - List dates = Arrays.asList(EARLY_DATE, EARLY_DATE, LATE_DATE); - mySvc.setDateFromPid(pid -> dates.get(pid.intValue() - 1)); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Arrays.asList(PID1, PID2, PID3)); - assertEquals(LATE_DATE, newThreshold); - assertThat(seenPids, contains(PID3)); - } - - @Test - public void threeItemsDifferentELL() { - List dates = Arrays.asList(EARLY_DATE, LATE_DATE, LATE_DATE); - mySvc.setDateFromPid(pid -> dates.get(pid.intValue() - 1)); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Arrays.asList(PID1, PID2, PID3)); - assertEquals(LATE_DATE, newThreshold); - assertThat(seenPids, contains(PID2, PID3)); - } - - - @Test - public void threeItemsDifferentLEE() { - List dates = Arrays.asList(LATE_DATE, EARLY_DATE, EARLY_DATE); - mySvc.setDateFromPid(pid -> dates.get(pid.intValue() - 1)); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Arrays.asList(PID1, PID2, PID3)); - assertEquals(EARLY_DATE, newThreshold); - assertThat(seenPids, contains(PID2, PID3)); - } - - @Test - public void threeItemsDifferentLLE() { - List dates = Arrays.asList(LATE_DATE, LATE_DATE, EARLY_DATE); - mySvc.setDateFromPid(pid -> dates.get(pid.intValue() - 1)); - Set seenPids = new HashSet<>(); - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, Arrays.asList(PID1, PID2, PID3)); - assertEquals(EARLY_DATE, newThreshold); - assertThat(seenPids, contains(PID3)); - } - - @Test - public void oneHundredItemsSameDate() { - mySvc.setDateFromPid(pid -> LATE_DATE); - Set seenPids = new HashSet<>(); - List bigList = new ArrayList<>(); - for (int i = 0; i < 100; ++i) { - bigList.add((long) i); - } - Date newThreshold = mySvc.updateThresholdAndCache(LATE_DATE, seenPids, bigList); - assertEquals(LATE_DATE, newThreshold); - assertThat(seenPids, hasSize(100)); - } -} diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReaderTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReaderTest.java deleted file mode 100644 index 6f0e0db249d..00000000000 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/batch/reader/ReverseCronologicalBatchResourcePidReaderTest.java +++ /dev/null @@ -1,173 +0,0 @@ -package ca.uhn.fhir.jpa.batch.reader; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.context.RuntimeResourceDefinition; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; -import ca.uhn.fhir.jpa.batch.job.model.PartitionedUrl; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; -import ca.uhn.fhir.jpa.dao.IResultIterator; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.jsonldjava.shaded.com.google.common.collect.Lists; -import org.hl7.fhir.r4.model.Patient; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.GregorianCalendar; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasSize; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class ReverseCronologicalBatchResourcePidReaderTest { - private static final int BATCH_SIZE = 3; - static FhirContext ourFhirContext = FhirContext.forR4Cached(); - static String URL_A = "a"; - static String URL_B = "b"; - static String URL_C = "c"; - static Set emptySet = Collections.emptySet(); - static RequestPartitionId partId = RequestPartitionId.defaultPartition(); - - Patient myPatient; - - @Mock - MatchUrlService myMatchUrlService; - @Mock - DaoRegistry myDaoRegistry; - @Mock - IFhirResourceDao myPatientDao; - private final RequestPartitionId myDefaultPartitionId = RequestPartitionId.defaultPartition(); - @Mock - private IResultIterator myResultIter; - - @InjectMocks - ReverseCronologicalBatchResourcePidReader myReader = new ReverseCronologicalBatchResourcePidReader(); - @Mock - private BatchResourceSearcher myBatchResourceSearcher; - - @BeforeEach - public void before() throws JsonProcessingException { - RequestListJson requestListJson = new RequestListJson(); - requestListJson.setPartitionedUrls(Lists.newArrayList(new PartitionedUrl(URL_A, partId), new PartitionedUrl(URL_B, partId), new PartitionedUrl(URL_C, partId))); - ObjectMapper mapper = new ObjectMapper(); - String requestListJsonString = mapper.writeValueAsString(requestListJson); - myReader.setRequestListJson(requestListJsonString); - myReader.setBatchSize(BATCH_SIZE); - - SearchParameterMap map = new SearchParameterMap(); - RuntimeResourceDefinition patientResDef = ourFhirContext.getResourceDefinition("Patient"); - when(myMatchUrlService.getResourceSearch(URL_A, myDefaultPartitionId)).thenReturn(new ResourceSearch(patientResDef, map, myDefaultPartitionId)); - when(myMatchUrlService.getResourceSearch(URL_B, myDefaultPartitionId)).thenReturn(new ResourceSearch(patientResDef, map, myDefaultPartitionId)); - when(myMatchUrlService.getResourceSearch(URL_C, myDefaultPartitionId)).thenReturn(new ResourceSearch(patientResDef, map, myDefaultPartitionId)); - when(myDaoRegistry.getResourceDao("Patient")).thenReturn(myPatientDao); - myPatient = new Patient(); - when(myPatientDao.readByPid(any())).thenReturn(myPatient); - Calendar cal = new GregorianCalendar(2021, 1, 1); - myPatient.getMeta().setLastUpdated(cal.getTime()); - - when(myBatchResourceSearcher.performSearch(any(), any())).thenReturn(myResultIter); - } - - private Set buildPidSet(Integer... thePids) { - return Arrays.stream(thePids) - .map(Long::new) - .map(ResourcePersistentId::new) - .collect(Collectors.toSet()); - } - - @Test - public void test3x1() throws Exception { - when(myResultIter.getNextResultBatch(BATCH_SIZE)) - .thenReturn(buildPidSet(1, 2, 3)) - .thenReturn(emptySet) - .thenReturn(buildPidSet(4, 5, 6)) - .thenReturn(emptySet) - .thenReturn(buildPidSet(7, 8)) - .thenReturn(emptySet); - - assertListEquals(myReader.read(), 1, 2, 3); - assertListEquals(myReader.read(), 4, 5, 6); - assertListEquals(myReader.read(), 7, 8); - assertNull(myReader.read()); - } - - @Test - public void testReadRepeat() throws Exception { - when(myResultIter.getNextResultBatch(BATCH_SIZE)) - .thenReturn(buildPidSet(1, 2, 3)) - .thenReturn(buildPidSet(1, 2, 3)) - .thenReturn(buildPidSet(2, 3, 4)) - .thenReturn(buildPidSet(4, 5)) - .thenReturn(emptySet); - - when(myResultIter.hasNext()) - .thenReturn(true) - .thenReturn(true) - .thenReturn(true) - .thenReturn(true) - .thenReturn(false); - - assertListEquals(myReader.read(), 1, 2, 3); - assertListEquals(myReader.read(), 4, 5); - assertNull(myReader.read()); - } - - @Test - public void test1x3start() throws Exception { - when(myResultIter.getNextResultBatch(BATCH_SIZE)) - .thenReturn(buildPidSet(1, 2, 3)) - .thenReturn(buildPidSet(4, 5, 6)) - .thenReturn(buildPidSet(7, 8)) - .thenReturn(emptySet) - .thenReturn(emptySet) - .thenReturn(emptySet); - - assertListEquals(myReader.read(), 1, 2, 3); - assertListEquals(myReader.read(), 4, 5, 6); - assertListEquals(myReader.read(), 7, 8); - assertNull(myReader.read()); - } - - @Test - public void test1x3end() throws Exception { - when(myResultIter.getNextResultBatch(BATCH_SIZE)) - .thenReturn(emptySet) - .thenReturn(emptySet) - .thenReturn(buildPidSet(1, 2, 3)) - .thenReturn(buildPidSet(4, 5, 6)) - .thenReturn(buildPidSet(7, 8)) - .thenReturn(emptySet); - - assertListEquals(myReader.read(), 1, 2, 3); - assertListEquals(myReader.read(), 4, 5, 6); - assertListEquals(myReader.read(), 7, 8); - assertNull(myReader.read()); - } - - private void assertListEquals(List theList, Integer... theValues) { - assertThat(theList, hasSize(theValues.length)); - for (int i = 0; i < theList.size(); ++i) { - assertEquals(theList.get(i), Long.valueOf(theValues[i])); - } - } -} diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeDaoTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeDaoTest.java index 60138e97333..42577ee41b2 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeDaoTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/dao/expunge/DeleteExpungeDaoTest.java @@ -1,13 +1,13 @@ package ca.uhn.fhir.jpa.dao.expunge; +import ca.uhn.fhir.batch2.model.JobInstance; +import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome; -import ca.uhn.fhir.jpa.batch.listener.PidReaderCounterListener; -import ca.uhn.fhir.jpa.batch.writer.SqlExecutorWriter; -import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.partition.SystemRequestDetails; +import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.server.exceptions.BaseServerResponseException; import ca.uhn.fhir.test.utilities.BatchJobHelper; @@ -21,8 +21,6 @@ import org.hl7.fhir.r4.model.Reference; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.batch.core.BatchStatus; -import org.springframework.batch.core.JobExecution; import org.springframework.beans.factory.annotation.Autowired; import java.util.List; @@ -90,7 +88,6 @@ class DeleteExpungeDaoTest extends BaseJpaR4Test { assertEquals(Msg.code(964) + "_expunge cannot be used with _cascade", e.getMessage()); } - @Test public void testDeleteExpungeThrowExceptionIfForeignKeyLinksExists() { // setup @@ -104,19 +101,19 @@ class DeleteExpungeDaoTest extends BaseJpaR4Test { // execute DeleteMethodOutcome outcome = myOrganizationDao.deleteByUrl("Organization?" + JpaConstants.PARAM_DELETE_EXPUNGE + "=true", mySrd); - Long jobExecutionId = jobExecutionIdFromOutcome(outcome); - JobExecution job = myBatchJobHelper.awaitJobExecution(jobExecutionId); + String jobExecutionId = jobExecutionIdFromOutcome(outcome); + JobInstance job = myBatch2JobHelper.awaitJobFailure(jobExecutionId); // validate - assertEquals(BatchStatus.FAILED, job.getStatus()); - assertThat(job.getExitStatus().getExitDescription(), containsString("DELETE with _expunge=true failed. Unable to delete " + organizationId.toVersionless() + " because " + patientId.toVersionless() + " refers to it via the path Patient.managingOrganization")); + assertEquals(StatusEnum.ERRORED, job.getStatus()); + assertThat(job.getErrorMessage(), containsString("DELETE with _expunge=true failed. Unable to delete " + organizationId.toVersionless() + " because " + patientId.toVersionless() + " refers to it via the path Patient.managingOrganization")); } - private Long jobExecutionIdFromOutcome(DeleteMethodOutcome theResult) { + private String jobExecutionIdFromOutcome(DeleteMethodOutcome theResult) { OperationOutcome operationOutcome = (OperationOutcome) theResult.getOperationOutcome(); String diagnostics = operationOutcome.getIssueFirstRep().getDiagnostics(); String[] parts = diagnostics.split("Delete job submitted with id "); - return Long.valueOf(parts[1]); + return parts[1]; } @Test @@ -139,12 +136,12 @@ class DeleteExpungeDaoTest extends BaseJpaR4Test { // execute DeleteMethodOutcome outcome = myOrganizationDao.deleteByUrl("Organization?" + JpaConstants.PARAM_DELETE_EXPUNGE + "=true", mySrd); - Long jobId = jobExecutionIdFromOutcome(outcome); - JobExecution job = myBatchJobHelper.awaitJobExecution(jobId); + String jobId = jobExecutionIdFromOutcome(outcome); + JobInstance job = myBatch2JobHelper.awaitJobFailure(jobId); // validate - assertEquals(BatchStatus.FAILED, job.getStatus()); - assertThat(job.getExitStatus().getExitDescription(), containsString("DELETE with _expunge=true failed. Unable to delete ")); + assertEquals(StatusEnum.ERRORED, job.getStatus()); + assertThat(job.getErrorMessage(), containsString("DELETE with _expunge=true failed. Unable to delete ")); } @Test @@ -160,15 +157,14 @@ class DeleteExpungeDaoTest extends BaseJpaR4Test { DeleteMethodOutcome outcome = myPatientDao.deleteByUrl("Patient?" + JpaConstants.PARAM_DELETE_EXPUNGE + "=true", mySrd); // validate - Long jobExecutionId = jobExecutionIdFromOutcome(outcome); - JobExecution job = myBatchJobHelper.awaitJobExecution(jobExecutionId); + String jobId = jobExecutionIdFromOutcome(outcome); + JobInstance job = myBatch2JobHelper.awaitJobCompletion(jobId); - // 10 / 3 rounded up = 4 - assertEquals(4, myBatchJobHelper.getReadCount(jobExecutionId)); - assertEquals(4, myBatchJobHelper.getWriteCount(jobExecutionId)); + assertEquals(10, myBatch2JobHelper.getCombinedRecordsProcessed(jobId)); - assertEquals(30, job.getExecutionContext().getLong(SqlExecutorWriter.ENTITY_TOTAL_UPDATED_OR_DELETED)); - assertEquals(10, job.getExecutionContext().getLong(PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED)); + // TODO KHS replace these with a report +// assertEquals(30, job.getExecutionContext().getLong(SqlExecutorWriter.ENTITY_TOTAL_UPDATED_OR_DELETED)); +// assertEquals(10, job.getExecutionContext().getLong(PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED)); } @Test @@ -183,13 +179,13 @@ class DeleteExpungeDaoTest extends BaseJpaR4Test { DeleteMethodOutcome outcome = myPatientDao.deleteByUrl("Patient?" + JpaConstants.PARAM_DELETE_EXPUNGE + "=true", mySrd); // validate - Long jobExecutionId = jobExecutionIdFromOutcome(outcome); - JobExecution job = myBatchJobHelper.awaitJobExecution(jobExecutionId); - assertEquals(1, myBatchJobHelper.getReadCount(jobExecutionId)); - assertEquals(1, myBatchJobHelper.getWriteCount(jobExecutionId)); + String jobId = jobExecutionIdFromOutcome(outcome); + JobInstance job = myBatch2JobHelper.awaitJobCompletion(jobId); + assertEquals(10, myBatch2JobHelper.getCombinedRecordsProcessed(jobId)); - assertEquals(30, job.getExecutionContext().getLong(SqlExecutorWriter.ENTITY_TOTAL_UPDATED_OR_DELETED)); - assertEquals(10, job.getExecutionContext().getLong(PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED)); + // TODO KHS replace these with a report +// assertEquals(30, job.getExecutionContext().getLong(SqlExecutorWriter.ENTITY_TOTAL_UPDATED_OR_DELETED)); +// assertEquals(10, job.getExecutionContext().getLong(PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED)); } @Test @@ -205,15 +201,15 @@ class DeleteExpungeDaoTest extends BaseJpaR4Test { //execute DeleteMethodOutcome outcome = myPatientDao.deleteByUrl("Patient?" + JpaConstants.PARAM_DELETE_EXPUNGE + "=true", mySrd); - Long jobExecutionId = jobExecutionIdFromOutcome(outcome); - JobExecution job = myBatchJobHelper.awaitJobExecution(jobExecutionId); + String jobId = jobExecutionIdFromOutcome(outcome); + JobInstance job = myBatch2JobHelper.awaitJobCompletion(jobId); // validate - assertEquals(1, myBatchJobHelper.getReadCount(jobExecutionId)); - assertEquals(1, myBatchJobHelper.getWriteCount(jobExecutionId)); + assertEquals(2, myBatch2JobHelper.getCombinedRecordsProcessed(jobId)); - assertEquals(7, job.getExecutionContext().getLong(SqlExecutorWriter.ENTITY_TOTAL_UPDATED_OR_DELETED)); - assertEquals(2, job.getExecutionContext().getLong(PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED)); + // TODO KHS replace these with a report +// assertEquals(7, job.getExecutionContext().getLong(SqlExecutorWriter.ENTITY_TOTAL_UPDATED_OR_DELETED)); +// assertEquals(2, job.getExecutionContext().getLong(PidReaderCounterListener.RESOURCE_TOTAL_PROCESSED)); } } diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobTest.java index 694515639e3..a0ed9f8f1bc 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/DeleteExpungeJobTest.java @@ -1,32 +1,27 @@ package ca.uhn.fhir.jpa.delete.job; -import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterUtil; -import ca.uhn.fhir.jpa.test.BaseJpaR4Test; +import ca.uhn.fhir.batch2.api.IJobCoordinator; +import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeAppCtx; +import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeJobParameters; +import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.test.utilities.BatchJobHelper; +import ca.uhn.fhir.jpa.test.BaseJpaR4Test; +import ca.uhn.fhir.jpa.test.Batch2JobHelper; import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.DiagnosticReport; import org.hl7.fhir.r4.model.Observation; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Reference; import org.junit.jupiter.api.Test; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import static org.junit.jupiter.api.Assertions.assertEquals; public class DeleteExpungeJobTest extends BaseJpaR4Test { @Autowired - private IBatchJobSubmitter myBatchJobSubmitter; + private IJobCoordinator myJobCoordinator; @Autowired - @Qualifier(BatchConstants.DELETE_EXPUNGE_JOB_NAME) - private Job myDeleteExpungeJob; - @Autowired - private BatchJobHelper myBatchJobHelper; + private Batch2JobHelper myBatch2JobHelper; @Test public void testDeleteExpunge() throws Exception { @@ -47,19 +42,33 @@ public class DeleteExpungeJobTest extends BaseJpaR4Test { obsInactive.setSubject(new Reference(pDelId)); IIdType oDelId = myObservationDao.create(obsInactive).getId().toUnqualifiedVersionless(); + DiagnosticReport diagActive = new DiagnosticReport(); + diagActive.setSubject(new Reference(pKeepId)); + IIdType dKeepId = myDiagnosticReportDao.create(diagActive).getId().toUnqualifiedVersionless(); + + DiagnosticReport diagInactive = new DiagnosticReport(); + diagInactive.setSubject(new Reference(pDelId)); + IIdType dDelId = myDiagnosticReportDao.create(diagInactive).getId().toUnqualifiedVersionless(); + // validate precondition assertEquals(2, myPatientDao.search(SearchParameterMap.newSynchronous()).size()); assertEquals(2, myObservationDao.search(SearchParameterMap.newSynchronous()).size()); + assertEquals(2, myDiagnosticReportDao.search(SearchParameterMap.newSynchronous()).size()); - JobParameters jobParameters = MultiUrlJobParameterUtil.buildJobParameters("Observation?subject.active=false", "Patient?active=false"); + DeleteExpungeJobParameters jobParameters = new DeleteExpungeJobParameters(); + jobParameters.addUrl("Observation?subject.active=false").addUrl("DiagnosticReport?subject.active=false"); + + JobInstanceStartRequest startRequest = new JobInstanceStartRequest(); + startRequest.setParameters(jobParameters); + startRequest.setJobDefinitionId(DeleteExpungeAppCtx.JOB_DELETE_EXPUNGE); // execute - JobExecution jobExecution = myBatchJobSubmitter.runJob(myDeleteExpungeJob, jobParameters); - - myBatchJobHelper.awaitJobCompletion(jobExecution); + String jobId = myJobCoordinator.startInstance(startRequest); + myBatch2JobHelper.awaitJobCompletion(jobId); // validate - assertEquals(1, myPatientDao.search(SearchParameterMap.newSynchronous()).size()); assertEquals(1, myObservationDao.search(SearchParameterMap.newSynchronous()).size()); + assertEquals(1, myDiagnosticReportDao.search(SearchParameterMap.newSynchronous()).size()); + assertEquals(2, myPatientDao.search(SearchParameterMap.newSynchronous()).size()); } } diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/MultiUrlJobParameterValidatorTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/MultiUrlJobParameterValidatorTest.java deleted file mode 100644 index d9d0f9162da..00000000000 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/job/MultiUrlJobParameterValidatorTest.java +++ /dev/null @@ -1,73 +0,0 @@ -package ca.uhn.fhir.jpa.delete.job; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterUtil; -import ca.uhn.fhir.jpa.batch.job.MultiUrlJobParameterValidator; -import ca.uhn.fhir.jpa.searchparam.MatchUrlService; -import ca.uhn.fhir.jpa.searchparam.ResourceSearch; -import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import com.fasterxml.jackson.core.JsonProcessingException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersInvalidException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class MultiUrlJobParameterValidatorTest { - static final FhirContext ourFhirContext = FhirContext.forR4Cached(); - - @Mock - MatchUrlService myMatchUrlService; - @Mock - DaoRegistry myDaoRegistry; - - MultiUrlJobParameterValidator mySvc; - - @BeforeEach - public void initMocks() { - mySvc = new MultiUrlJobParameterValidator(myMatchUrlService, myDaoRegistry); - } - - @Test - public void testValidate() throws JobParametersInvalidException, JsonProcessingException { - // setup - JobParameters parameters = MultiUrlJobParameterUtil.buildJobParameters("Patient?address=memory", "Patient?name=smith"); - ResourceSearch resourceSearch = new ResourceSearch(ourFhirContext.getResourceDefinition("Patient"), new SearchParameterMap(), RequestPartitionId.defaultPartition()); - when(myMatchUrlService.getResourceSearch(anyString(), any())).thenReturn(resourceSearch); - when(myDaoRegistry.isResourceTypeSupported("Patient")).thenReturn(true); - - // execute - mySvc.validate(parameters); - // verify - verify(myMatchUrlService, times(2)).getResourceSearch(anyString(), any()); - } - - @Test - public void testValidateBadType() throws JobParametersInvalidException, JsonProcessingException { - JobParameters parameters = MultiUrlJobParameterUtil.buildJobParameters("Patient?address=memory"); - ResourceSearch resourceSearch = new ResourceSearch(ourFhirContext.getResourceDefinition("Patient"), new SearchParameterMap(), RequestPartitionId.defaultPartition()); - when(myMatchUrlService.getResourceSearch(anyString(), any())).thenReturn(resourceSearch); - when(myDaoRegistry.isResourceTypeSupported("Patient")).thenReturn(false); - - try { - mySvc.validate(parameters); - fail(); - } catch (JobParametersInvalidException e) { - assertEquals(Msg.code(1281) + "The resource type Patient is not supported on this server.", e.getMessage()); - } - } -} diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/provider/DeleteExpungeProviderTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/provider/DeleteExpungeProviderTest.java index fe3ce53e4f3..e346c029613 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/provider/DeleteExpungeProviderTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/delete/provider/DeleteExpungeProviderTest.java @@ -1,9 +1,9 @@ package ca.uhn.fhir.jpa.delete.provider; +import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeProvider; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; import ca.uhn.fhir.rest.server.RestfulServer; -import ca.uhn.fhir.rest.server.provider.DeleteExpungeProvider; import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.test.utilities.JettyUtil; import org.apache.commons.io.IOUtils; diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java index ba9da3a14a9..f8afe15c9f5 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/BaseResourceProviderR4Test.java @@ -1,10 +1,11 @@ package ca.uhn.fhir.jpa.provider.r4; +import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeProvider; +import ca.uhn.fhir.batch2.jobs.reindex.ReindexProvider; import ca.uhn.fhir.context.support.IValidationSupport; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; import ca.uhn.fhir.jpa.dao.data.IPartitionDao; -import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.graphql.GraphQLProvider; import ca.uhn.fhir.jpa.provider.DiffProvider; import ca.uhn.fhir.jpa.provider.JpaCapabilityStatementProvider; @@ -14,6 +15,7 @@ import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.searchparam.registry.SearchParamRegistryImpl; import ca.uhn.fhir.jpa.subscription.match.config.WebsocketDispatcherConfig; import ca.uhn.fhir.jpa.subscription.match.registry.SubscriptionLoader; +import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.util.ResourceCountCache; import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator; import ca.uhn.fhir.parser.StrictErrorHandler; @@ -24,8 +26,6 @@ import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; import ca.uhn.fhir.rest.server.RestfulServer; import ca.uhn.fhir.rest.server.interceptor.CorsInterceptor; import ca.uhn.fhir.rest.server.interceptor.ResponseHighlighterInterceptor; -import ca.uhn.fhir.rest.server.provider.DeleteExpungeProvider; -import ca.uhn.fhir.batch2.jobs.reindex.ReindexProvider; import ca.uhn.fhir.test.utilities.JettyUtil; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java index 3963241543d..b154e21949e 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java @@ -7,13 +7,11 @@ import ca.uhn.fhir.interceptor.api.IPointcut; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; import ca.uhn.fhir.jpa.delete.job.ReindexTestHelper; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; -import ca.uhn.fhir.test.utilities.BatchJobHelper; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; import org.hl7.fhir.r4.model.Bundle; @@ -25,7 +23,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; import java.util.List; @@ -39,9 +36,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class MultitenantBatchOperationR4Test extends BaseMultitenantResourceProviderR4Test { private static final Logger ourLog = LoggerFactory.getLogger(MultitenantBatchOperationR4Test.class); - @Autowired - private BatchJobHelper myBatchJobHelper; - @BeforeEach @Override public void before() throws Exception { @@ -82,7 +76,7 @@ public class MultitenantBatchOperationR4Test extends BaseMultitenantResourceProv assertEquals(0, getAllPatientsInTenant(DEFAULT_PARTITION_NAME).getTotal()); Parameters input = new Parameters(); - input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "/Patient?active=false"); + input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "Patient?active=false"); MyInterceptor interceptor = new MyInterceptor(); myInterceptorRegistry.registerAnonymousInterceptor(Pointcut.STORAGE_PARTITION_SELECTED, interceptor); @@ -97,8 +91,10 @@ public class MultitenantBatchOperationR4Test extends BaseMultitenantResourceProv .execute(); ourLog.info(myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(response)); - myBatchJobHelper.awaitAllBulkJobCompletions(BatchConstants.DELETE_EXPUNGE_JOB_NAME); - assertThat(interceptor.requestPartitionIds, hasSize(1)); + String jobId = BatchHelperR4.jobIdFromBatch2Parameters(response); + myBatch2JobHelper.awaitSingleChunkJobCompletion(jobId); + + assertThat(interceptor.requestPartitionIds, hasSize(3)); RequestPartitionId partitionId = interceptor.requestPartitionIds.get(0); assertEquals(TENANT_B_ID, partitionId.getFirstPartitionIdOrNull()); assertEquals(TENANT_B, partitionId.getFirstPartitionNameOrNull()); @@ -106,10 +102,7 @@ public class MultitenantBatchOperationR4Test extends BaseMultitenantResourceProv assertEquals("Patient", interceptor.resourceDefs.get(0).getName()); myInterceptorRegistry.unregisterInterceptor(interceptor); - Long jobId = BatchHelperR4.jobIdFromParameters(response); - - assertEquals(1, myBatchJobHelper.getReadCount(jobId)); - assertEquals(1, myBatchJobHelper.getWriteCount(jobId)); + assertEquals(1, myBatch2JobHelper.getCombinedRecordsProcessed(jobId)); // validate only the false patient in TENANT_B is removed assertEquals(2, getAllPatientsInTenant(TENANT_A).getTotal()); diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java index 50209aa7170..1cef351257e 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/provider/r4/SystemProviderR4Test.java @@ -1,12 +1,11 @@ package ca.uhn.fhir.jpa.provider.r4; +import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeProvider; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.api.Hook; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.provider.SystemProviderDstu2Test; import ca.uhn.fhir.jpa.rp.r4.BinaryResourceProvider; import ca.uhn.fhir.jpa.rp.r4.DiagnosticReportResourceProvider; @@ -17,6 +16,7 @@ import ca.uhn.fhir.jpa.rp.r4.PatientResourceProvider; import ca.uhn.fhir.jpa.rp.r4.PractitionerResourceProvider; import ca.uhn.fhir.jpa.rp.r4.ServiceRequestResourceProvider; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.EncodingEnum; @@ -32,7 +32,6 @@ import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceGoneException; import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor; import ca.uhn.fhir.rest.server.interceptor.ResponseHighlighterInterceptor; -import ca.uhn.fhir.rest.server.provider.DeleteExpungeProvider; import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.test.utilities.BatchJobHelper; import ca.uhn.fhir.test.utilities.JettyUtil; @@ -62,6 +61,7 @@ import org.hl7.fhir.r4.model.Bundle.BundleType; import org.hl7.fhir.r4.model.Bundle.HTTPVerb; import org.hl7.fhir.r4.model.CodeType; import org.hl7.fhir.r4.model.DecimalType; +import org.hl7.fhir.r4.model.DiagnosticReport; import org.hl7.fhir.r4.model.Enumerations.AdministrativeGender; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Observation; @@ -822,13 +822,22 @@ public class SystemProviderR4Test extends BaseJpaR4Test { obsInactive.setSubject(new Reference(pDelId.toUnqualifiedVersionless())); IIdType obsDelId = myClient.create().resource(obsInactive).execute().getId(); + DiagnosticReport diagActive = new DiagnosticReport(); + diagActive.setSubject(new Reference(pKeepId.toUnqualifiedVersionless())); + IIdType dKeepId = myClient.create().resource(diagActive).execute().getId(); + + DiagnosticReport diagInactive = new DiagnosticReport(); + diagInactive.setSubject(new Reference(pDelId.toUnqualifiedVersionless())); + IIdType diagDelId = myClient.create().resource(diagInactive).execute().getId(); + // validate setup assertEquals(14, getAllResourcesOfType("Patient").getTotal()); assertEquals(2, getAllResourcesOfType("Observation").getTotal()); + assertEquals(2, getAllResourcesOfType("DiagnosticReport").getTotal()); Parameters input = new Parameters(); - input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "/Observation?subject.active=false"); - input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "/Patient?active=false"); + input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "Observation?subject.active=false"); + input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "DiagnosticReport?subject.active=false"); int batchSize = 2; input.addParameter(ProviderConstants.OPERATION_DELETE_BATCH_SIZE, new DecimalType(batchSize)); @@ -842,18 +851,13 @@ public class SystemProviderR4Test extends BaseJpaR4Test { .execute(); ourLog.info(ourCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(response)); - myBatchJobHelper.awaitAllBulkJobCompletions(BatchConstants.DELETE_EXPUNGE_JOB_NAME); - Long jobId = BatchHelperR4.jobIdFromParameters(response); + String jobId = BatchHelperR4.jobIdFromBatch2Parameters(response); // validate - // 1 observation - // + 12/batchSize inactive patients - // + 1 patient with id pDelId - // = 1 + 6 + 1 = 8 - assertEquals(8, myBatchJobHelper.getReadCount(jobId)); - assertEquals(8, myBatchJobHelper.getWriteCount(jobId)); + myBatch2JobHelper.awaitJobCompletion(jobId); + assertEquals(2, myBatch2JobHelper.getCombinedRecordsProcessed(jobId)); // validate Bundle obsBundle = getAllResourcesOfType("Observation"); @@ -861,11 +865,10 @@ public class SystemProviderR4Test extends BaseJpaR4Test { assertThat(observations, hasSize(1)); assertEquals(oKeepId, observations.get(0).getIdElement()); - Bundle patientBundle = getAllResourcesOfType("Patient"); - List patients = BundleUtil.toListOfResourcesOfType(myFhirContext, patientBundle, Patient.class); - assertThat(patients, hasSize(1)); - assertEquals(pKeepId, patients.get(0).getIdElement()); - + Bundle diagBundle = getAllResourcesOfType("DiagnosticReport"); + List diags = BundleUtil.toListOfResourcesOfType(myFhirContext, diagBundle, DiagnosticReport.class); + assertThat(diags, hasSize(1)); + assertEquals(dKeepId, diags.get(0).getIdElement()); } private Bundle getAllResourcesOfType(String theResourceName) { diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java index 28c5992bf9c..b2e6706e7e3 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/reindex/ResourceReindexSvcImplTest.java @@ -2,7 +2,7 @@ package ca.uhn.fhir.jpa.reindex; import ca.uhn.fhir.jpa.api.pid.IResourcePidList; import ca.uhn.fhir.jpa.api.pid.TypedResourcePid; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; @@ -24,7 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class ResourceReindexSvcImplTest extends BaseJpaR4Test { @Autowired - private IResourceReindexSvc mySvc; + private IBatch2DaoSvc mySvc; @Test public void testFetchResourceIdsPage_NoUrl_WithData() { diff --git a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java index dfca2e9ee9f..09aa623aef5 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java +++ b/hapi-fhir-jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/stresstest/StressTestR4Test.java @@ -2,12 +2,11 @@ package ca.uhn.fhir.jpa.stresstest; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.test.config.TestR4Config; import ca.uhn.fhir.jpa.provider.r4.BaseResourceProviderR4Test; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; +import ca.uhn.fhir.jpa.test.config.TestR4Config; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.MethodOutcome; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -28,6 +27,7 @@ import org.hamcrest.Matchers; import org.hl7.fhir.common.hapi.validation.validator.FhirInstanceValidator; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.Bundle.BundleType; import org.hl7.fhir.r4.model.Bundle.HTTPVerb; @@ -586,11 +586,12 @@ public class StressTestR4Test extends BaseResourceProviderR4Test { Parameters input = new Parameters(); - input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "/Patient?active=true"); + input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, "Patient?active=true"); int batchSize = 2; input.addParameter(ProviderConstants.OPERATION_DELETE_BATCH_SIZE, new DecimalType(batchSize)); // execute + myCaptureQueriesListener.clear(); Parameters response = myClient .operation() .onServer() @@ -599,11 +600,13 @@ public class StressTestR4Test extends BaseResourceProviderR4Test { .execute(); ourLog.info(myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(response)); - myBatchJobHelper.awaitAllBulkJobCompletions(BatchConstants.DELETE_EXPUNGE_JOB_NAME); - int deleteCount = myCaptureQueriesListener.countDeleteQueries(); + + String jobId = BatchHelperR4.jobIdFromBatch2Parameters(response); + myBatch2JobHelper.awaitJobCompletion(jobId); + int deleteCount = myCaptureQueriesListener.getDeleteQueries().size(); myCaptureQueriesListener.logDeleteQueries(); - assertThat(deleteCount, is(equalTo(19))); + assertThat(deleteCount, is(equalTo(30))); } private void validateNoErrors(List tasks) { diff --git a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml index 08fc2ef31cc..de68b19745d 100644 --- a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml +++ b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-server-mdm/pom.xml b/hapi-fhir-server-mdm/pom.xml index 542dfa9e1f8..247a15483d0 100644 --- a/hapi-fhir-server-mdm/pom.xml +++ b/hapi-fhir-server-mdm/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server-openapi/pom.xml b/hapi-fhir-server-openapi/pom.xml index d69a8163bda..91c80f10c79 100644 --- a/hapi-fhir-server-openapi/pom.xml +++ b/hapi-fhir-server-openapi/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server/pom.xml b/hapi-fhir-server/pom.xml index 4d2fdba8523..54c74d414f7 100644 --- a/hapi-fhir-server/pom.xml +++ b/hapi-fhir-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IDeleteExpungeJobSubmitter.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IDeleteExpungeJobSubmitter.java index 78bfb92f828..ff457d64c3c 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IDeleteExpungeJobSubmitter.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IDeleteExpungeJobSubmitter.java @@ -20,6 +20,16 @@ package ca.uhn.fhir.rest.api.server.storage; * #L% */ +import ca.uhn.fhir.rest.api.server.RequestDetails; + +import java.util.List; + // Tag interface for Spring auto-wiring -public interface IDeleteExpungeJobSubmitter extends IMultiUrlJobSubmitter { +public interface IDeleteExpungeJobSubmitter { + /** + * @param theBatchSize For each pass, when synchronously searching for resources, limit the number of matching resources to this number + * @param theUrlsToProcess A list of strings of the form "/Patient?active=true" + * @return The Batch2 JobId that was started to run this batch job + */ + String submitJob(Integer theBatchSize, List theUrlsToProcess, RequestDetails theRequest); } diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IMultiUrlJobSubmitter.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IMultiUrlJobSubmitter.java deleted file mode 100644 index 87a3fa35181..00000000000 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IMultiUrlJobSubmitter.java +++ /dev/null @@ -1,37 +0,0 @@ -package ca.uhn.fhir.rest.api.server.storage; - -/*- - * #%L - * HAPI FHIR - Server Framework - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.rest.api.server.RequestDetails; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParametersInvalidException; - -import java.util.List; - -public interface IMultiUrlJobSubmitter { - /** - * @param theBatchSize For each pass, when synchronously searching for resources, limit the number of matching resources to this number - * @param theUrlsToProcess A list of strings of the form "/Patient?active=true" - * @return The Spring Batch JobExecution that was started to run this batch job - * @throws JobParametersInvalidException - */ - JobExecution submitJob(Integer theBatchSize, List theUrlsToProcess, RequestDetails theRequest) throws JobParametersInvalidException; -} diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IReindexJobSubmitter.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IReindexJobSubmitter.java deleted file mode 100644 index e562ab55d13..00000000000 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/storage/IReindexJobSubmitter.java +++ /dev/null @@ -1,30 +0,0 @@ -package ca.uhn.fhir.rest.api.server.storage; - -/*- - * #%L - * HAPI FHIR - Server Framework - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.rest.api.server.RequestDetails; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParametersInvalidException; - -// Tag interface for Spring wiring -public interface IReindexJobSubmitter extends IMultiUrlJobSubmitter { - JobExecution submitEverythingJob(Integer theBatchSize, RequestDetails theRequest) throws JobParametersInvalidException; -} diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/MultiUrlProcessor.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/MultiUrlProcessor.java deleted file mode 100644 index 074874c25d6..00000000000 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/MultiUrlProcessor.java +++ /dev/null @@ -1,66 +0,0 @@ -package ca.uhn.fhir.rest.server.provider; - -/*- - * #%L - * HAPI FHIR - Server Framework - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.api.server.storage.IMultiUrlJobSubmitter; -import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; -import ca.uhn.fhir.util.ParametersUtil; -import org.hl7.fhir.instance.model.api.IBaseParameters; -import org.hl7.fhir.instance.model.api.IPrimitiveType; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParametersInvalidException; - -import javax.annotation.Nullable; -import java.math.BigDecimal; -import java.util.List; - -public class MultiUrlProcessor { - private final FhirContext myFhirContext; - private final IMultiUrlJobSubmitter myMultiUrlProcessorJobSubmitter; - - public MultiUrlProcessor(FhirContext theFhirContext, IMultiUrlJobSubmitter theMultiUrlProcessorJobSubmitter) { - myMultiUrlProcessorJobSubmitter = theMultiUrlProcessorJobSubmitter; - myFhirContext = theFhirContext; - } - - public IBaseParameters processUrls(List theUrlsToProcess, Integer theBatchSize, RequestDetails theRequestDetails) { - try { - JobExecution jobExecution = myMultiUrlProcessorJobSubmitter.submitJob(theBatchSize, theUrlsToProcess, theRequestDetails); - IBaseParameters retval = ParametersUtil.newInstance(myFhirContext); - ParametersUtil.addParameterToParametersLong(myFhirContext, retval, ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, jobExecution.getJobId()); - return retval; - } catch (JobParametersInvalidException e) { - throw new InvalidRequestException(Msg.code(309) + "Invalid job parameters: " + e.getMessage(), e); - } - } - - @Nullable - public Integer getBatchSize(IPrimitiveType theBatchSize) { - Integer batchSize = null; - if (theBatchSize != null && !theBatchSize.isEmpty()) { - batchSize = theBatchSize.getValue().intValue(); - } - return batchSize; - } -} diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml index cb05689ecbe..64843b89e85 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml index a742e69d87e..9db36c00196 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT hapi-fhir-spring-boot-sample-client-apache diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml index d7bc02cccd9..ae97630d10e 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT hapi-fhir-spring-boot-sample-client-okhttp diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml index ab84da3e38b..7b601e60bbb 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT hapi-fhir-spring-boot-sample-server-jersey diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml index 7408fddec99..6ce31703d0d 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT hapi-fhir-spring-boot-samples diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml index b2b0e5c1603..64851521d74 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/pom.xml b/hapi-fhir-spring-boot/pom.xml index 1e8a9855073..32caa2604dc 100644 --- a/hapi-fhir-spring-boot/pom.xml +++ b/hapi-fhir-spring-boot/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-sql-migrate/pom.xml b/hapi-fhir-sql-migrate/pom.xml index 499471d8307..5c4cb62bc98 100644 --- a/hapi-fhir-sql-migrate/pom.xml +++ b/hapi-fhir-sql-migrate/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-batch2-jobs/pom.xml b/hapi-fhir-storage-batch2-jobs/pom.xml index d31928085fe..d133f53c918 100644 --- a/hapi-fhir-storage-batch2-jobs/pom.xml +++ b/hapi-fhir-storage-batch2-jobs/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java index dd56dedceaa..874b11586b0 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/Batch2JobsConfig.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.batch2.jobs.config; * #L% */ +import ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeAppCtx; import ca.uhn.fhir.batch2.jobs.imprt.BulkImportAppCtx; import ca.uhn.fhir.batch2.jobs.reindex.ReindexAppCtx; import org.springframework.context.annotation.Configuration; @@ -28,8 +29,10 @@ import org.springframework.context.annotation.Import; //When you define a new batch job, add it here. @Configuration @Import({ + BatchCommonCtx.class, BulkImportAppCtx.class, - ReindexAppCtx.class + ReindexAppCtx.class, + DeleteExpungeAppCtx.class }) public class Batch2JobsConfig { // nothing diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/BatchCommonCtx.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/BatchCommonCtx.java new file mode 100644 index 00000000000..cce7902ce68 --- /dev/null +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/config/BatchCommonCtx.java @@ -0,0 +1,13 @@ +package ca.uhn.fhir.batch2.jobs.config; + +import ca.uhn.fhir.batch2.jobs.parameters.UrlPartitioner; +import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; +import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import org.springframework.context.annotation.Bean; + +public class BatchCommonCtx { + @Bean + UrlPartitioner urlPartitioner(MatchUrlService theMatchUrlService, IRequestPartitionHelperSvc theRequestPartitionHelperSvc) { + return new UrlPartitioner(theMatchUrlService, theRequestPartitionHelperSvc); + } +} diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeAppCtx.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeAppCtx.java new file mode 100644 index 00000000000..cf7377e7aa3 --- /dev/null +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeAppCtx.java @@ -0,0 +1,99 @@ +package ca.uhn.fhir.batch2.jobs.expunge; + +/*- + * #%L + * hapi-fhir-storage-batch2-jobs + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; +import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; +import ca.uhn.fhir.batch2.jobs.parameters.UrlListValidator; +import ca.uhn.fhir.batch2.jobs.step.GenerateRangeChunksStep; +import ca.uhn.fhir.batch2.jobs.step.LoadIdsStep; +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; +import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; +import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; +import ca.uhn.fhir.rest.server.provider.ProviderConstants; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class DeleteExpungeAppCtx { + + public static final String JOB_DELETE_EXPUNGE = "DELETE_EXPUNGE"; + + + @Bean + public JobDefinition expungeJobDefinition( + IBatch2DaoSvc theBatch2DaoSvc, + HapiTransactionService theHapiTransactionService, + IDeleteExpungeSvc theDeleteExpungeSvc + ) { + return JobDefinition + .newBuilder() + .setJobDefinitionId(JOB_DELETE_EXPUNGE) + .setJobDescription("Expunge resources") + .setJobDefinitionVersion(1) + .setParametersType(DeleteExpungeJobParameters.class) + .setParametersValidator(expungeJobParametersValidator(theBatch2DaoSvc)) + .gatedExecution() + .addFirstStep( + "generate-ranges", + "Generate data ranges to expunge", + PartitionedUrlChunkRangeJson.class, + expungeGenerateRangeChunksStep()) + .addIntermediateStep( + "load-ids", + "Load IDs of resources to expunge", + ResourceIdListWorkChunkJson.class, + loadIdsStep(theBatch2DaoSvc)) + .addLastStep("expunge", + "Perform the resource expunge", + expungeStep(theHapiTransactionService, theDeleteExpungeSvc) + ) + .build(); + } + + @Bean + public DeleteExpungeJobParametersValidator expungeJobParametersValidator(IBatch2DaoSvc theBatch2DaoSvc) { + return new DeleteExpungeJobParametersValidator(new UrlListValidator(ProviderConstants.OPERATION_EXPUNGE, theBatch2DaoSvc)); + } + + @Bean + public DeleteExpungeStep expungeStep(HapiTransactionService theHapiTransactionService, IDeleteExpungeSvc theDeleteExpungeSvc) { + return new DeleteExpungeStep(theHapiTransactionService, theDeleteExpungeSvc); + } + + @Bean + public GenerateRangeChunksStep expungeGenerateRangeChunksStep() { + return new GenerateRangeChunksStep(); + } + + @Bean + public LoadIdsStep loadIdsStep(IBatch2DaoSvc theBatch2DaoSvc) { + return new LoadIdsStep(theBatch2DaoSvc); + } + + @Bean + public DeleteExpungeProvider deleteExpungeProvider(FhirContext theFhirContext, IDeleteExpungeJobSubmitter theDeleteExpungeJobSubmitter) { + return new DeleteExpungeProvider(theFhirContext, theDeleteExpungeJobSubmitter); + } +} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParameters.java similarity index 73% rename from hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java rename to hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParameters.java index 4d637218620..243b6a7ce1b 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmBatchJobSubmitterFactory.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParameters.java @@ -1,8 +1,8 @@ -package ca.uhn.fhir.mdm.api; +package ca.uhn.fhir.batch2.jobs.expunge; /*- * #%L - * HAPI FHIR - Master Data Management + * hapi-fhir-storage-batch2-jobs * %% * Copyright (C) 2014 - 2022 Smile CDR, Inc. * %% @@ -20,6 +20,7 @@ package ca.uhn.fhir.mdm.api; * #L% */ -public interface IMdmBatchJobSubmitterFactory { - IMdmClearJobSubmitter getClearJobSubmitter(); +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrlListJobParameters; + +public class DeleteExpungeJobParameters extends PartitionedUrlListJobParameters { } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParametersValidator.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParametersValidator.java new file mode 100644 index 00000000000..d0cedc9f676 --- /dev/null +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobParametersValidator.java @@ -0,0 +1,42 @@ +package ca.uhn.fhir.batch2.jobs.expunge; + +/*- + * #%L + * hapi-fhir-storage-batch2-jobs + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.batch2.api.IJobParametersValidator; +import ca.uhn.fhir.batch2.jobs.parameters.UrlListValidator; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.List; + +public class DeleteExpungeJobParametersValidator implements IJobParametersValidator { + private final UrlListValidator myUrlListValidator; + + public DeleteExpungeJobParametersValidator(UrlListValidator theUrlListValidator) { + myUrlListValidator = theUrlListValidator; + } + + @Nullable + @Override + public List validate(@NotNull DeleteExpungeJobParameters theParameters) { + return myUrlListValidator.validatePartitionedUrls(theParameters.getPartitionedUrls()); + } +} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteExpungeJobSubmitterImpl.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobSubmitterImpl.java similarity index 53% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteExpungeJobSubmitterImpl.java rename to hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobSubmitterImpl.java index a7b8161e86c..b70387255dc 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/delete/DeleteExpungeJobSubmitterImpl.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeJobSubmitterImpl.java @@ -1,8 +1,8 @@ -package ca.uhn.fhir.jpa.delete; +package ca.uhn.fhir.batch2.jobs.expunge; /*- * #%L - * HAPI FHIR JPA Server + * hapi-fhir-storage-batch2-jobs * %% * Copyright (C) 2014 - 2022 Smile CDR, Inc. * %% @@ -20,41 +20,36 @@ package ca.uhn.fhir.jpa.delete; * #L% */ -import ca.uhn.fhir.i18n.Msg; +import ca.uhn.fhir.batch2.api.IJobCoordinator; +import ca.uhn.fhir.batch2.jobs.parameters.UrlPartitioner; +import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.api.HookParams; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.api.Pointcut; +import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails; +import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.config.DaoConfig; -import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter; -import ca.uhn.fhir.jpa.batch.config.BatchConstants; -import ca.uhn.fhir.jpa.batch.job.PartitionedUrlValidator; -import ca.uhn.fhir.jpa.batch.job.model.RequestListJson; -import ca.uhn.fhir.jpa.batch.reader.ReverseCronologicalBatchResourcePidReader; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import ca.uhn.fhir.rest.api.RestOperationTypeEnum; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; import ca.uhn.fhir.rest.server.exceptions.ForbiddenOperationException; -import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersInvalidException; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import javax.transaction.Transactional; import java.util.List; +import static ca.uhn.fhir.batch2.jobs.expunge.DeleteExpungeAppCtx.JOB_DELETE_EXPUNGE; + public class DeleteExpungeJobSubmitterImpl implements IDeleteExpungeJobSubmitter { @Autowired - private IBatchJobSubmitter myBatchJobSubmitter; - @Autowired - @Qualifier(BatchConstants.DELETE_EXPUNGE_JOB_NAME) - private Job myDeleteExpungeJob; + IJobCoordinator myJobCoordinator; @Autowired FhirContext myFhirContext; @Autowired @@ -64,30 +59,44 @@ public class DeleteExpungeJobSubmitterImpl implements IDeleteExpungeJobSubmitter @Autowired DaoConfig myDaoConfig; @Autowired - PartitionedUrlValidator myPartitionedUrlValidator; - @Autowired IInterceptorBroadcaster myInterceptorBroadcaster; + @Autowired + UrlPartitioner myUrlPartitioner; @Override @Transactional(Transactional.TxType.NEVER) - public JobExecution submitJob(Integer theBatchSize, List theUrlsToDeleteExpunge, RequestDetails theRequest) throws JobParametersInvalidException { + public String submitJob(Integer theBatchSize, List theUrlsToDeleteExpunge, RequestDetails theRequestDetails) { if (theBatchSize == null) { theBatchSize = myDaoConfig.getExpungeBatchSize(); } - RequestListJson requestListJson = myPartitionedUrlValidator.buildRequestListJson(theRequest, theUrlsToDeleteExpunge); if (!myDaoConfig.canDeleteExpunge()) { throw new ForbiddenOperationException(Msg.code(820) + "Delete Expunge not allowed: " + myDaoConfig.cannotDeleteExpungeReason()); } for (String url : theUrlsToDeleteExpunge) { HookParams params = new HookParams() - .add(RequestDetails.class, theRequest) - .addIfMatchesType(ServletRequestDetails.class, theRequest) + .add(RequestDetails.class, theRequestDetails) + .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) .add(String.class, url); - CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PRE_DELETE_EXPUNGE, params); + CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequestDetails, Pointcut.STORAGE_PRE_DELETE_EXPUNGE, params); } - JobParameters jobParameters = ReverseCronologicalBatchResourcePidReader.buildJobParameters(ProviderConstants.OPERATION_DELETE_EXPUNGE, theBatchSize, requestListJson); - return myBatchJobSubmitter.runJob(myDeleteExpungeJob, jobParameters); + DeleteExpungeJobParameters deleteExpungeJobParameters = new DeleteExpungeJobParameters(); + // Set partition for each url since resource type can determine partition + theUrlsToDeleteExpunge.stream() + .filter(StringUtils::isNotBlank) + .map(url -> myUrlPartitioner.partitionUrl(url, theRequestDetails)) + .forEach(deleteExpungeJobParameters::addPartitionedUrl); + deleteExpungeJobParameters.setBatchSize(theBatchSize); + + ReadPartitionIdRequestDetails details = new ReadPartitionIdRequestDetails(null, RestOperationTypeEnum.EXTENDED_OPERATION_SERVER, null, null, null); + // Also set toplevel partition in case there are no urls + RequestPartitionId requestPartition = myRequestPartitionHelperSvc.determineReadPartitionForRequest(theRequestDetails, null, details); + deleteExpungeJobParameters.setRequestPartitionId(requestPartition); + + JobInstanceStartRequest startRequest = new JobInstanceStartRequest(); + startRequest.setJobDefinitionId(JOB_DELETE_EXPUNGE); + startRequest.setParameters(deleteExpungeJobParameters); + return myJobCoordinator.startInstance(startRequest); } } diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/DeleteExpungeProvider.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeProvider.java similarity index 62% rename from hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/DeleteExpungeProvider.java rename to hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeProvider.java index 395a1bd4149..56c88079df4 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/server/provider/DeleteExpungeProvider.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeProvider.java @@ -1,8 +1,8 @@ -package ca.uhn.fhir.rest.server.provider; +package ca.uhn.fhir.batch2.jobs.expunge; /*- * #%L - * HAPI FHIR - Server Framework + * hapi-fhir-storage-batch2-jobs * %% * Copyright (C) 2014 - 2022 Smile CDR, Inc. * %% @@ -27,6 +27,9 @@ import ca.uhn.fhir.rest.annotation.OperationParam; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.provider.ProviderConstants; +import ca.uhn.fhir.util.ParametersUtil; +import org.apache.commons.lang3.StringUtils; import org.hl7.fhir.instance.model.api.IBaseParameters; import org.hl7.fhir.instance.model.api.IPrimitiveType; @@ -35,10 +38,12 @@ import java.util.List; import java.util.stream.Collectors; public class DeleteExpungeProvider { - private final MultiUrlProcessor myMultiUrlProcessor; + private final FhirContext myFhirContext; + private final IDeleteExpungeJobSubmitter myDeleteExpungeJobSubmitter; public DeleteExpungeProvider(FhirContext theFhirContext, IDeleteExpungeJobSubmitter theDeleteExpungeJobSubmitter) { - myMultiUrlProcessor = new MultiUrlProcessor(theFhirContext, theDeleteExpungeJobSubmitter); + myFhirContext = theFhirContext; + myDeleteExpungeJobSubmitter = theDeleteExpungeJobSubmitter; } @Operation(name = ProviderConstants.OPERATION_DELETE_EXPUNGE, idempotent = false) @@ -48,10 +53,21 @@ public class DeleteExpungeProvider { RequestDetails theRequestDetails ) { if (theUrlsToDeleteExpunge == null) { - throw new InvalidRequestException(Msg.code(1976) + "At least one `url` parameter to $delete-expunge must be provided."); + throw new InvalidRequestException(Msg.code(2101) + "At least one `url` parameter to $delete-expunge must be provided."); } - List urls = theUrlsToDeleteExpunge.stream().map(IPrimitiveType::getValue).collect(Collectors.toList()); - Integer batchSize = myMultiUrlProcessor.getBatchSize(theBatchSize); - return myMultiUrlProcessor.processUrls(urls, batchSize, theRequestDetails); + List urls = theUrlsToDeleteExpunge.stream() + .map(IPrimitiveType::getValue) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + + Integer batchSize = null; + if (theBatchSize != null && theBatchSize.getValue() !=null && theBatchSize.getValue().intValue() > 0) { + batchSize = theBatchSize.getValue().intValue(); + } + String jobId = myDeleteExpungeJobSubmitter.submitJob(batchSize, urls, theRequestDetails); + + IBaseParameters retval = ParametersUtil.newInstance(myFhirContext); + ParametersUtil.addParameterToParametersString(myFhirContext, retval, ProviderConstants.OPERATION_BATCH_RESPONSE_JOB_ID, jobId); + return retval; } } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java new file mode 100644 index 00000000000..42eb0382b4b --- /dev/null +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeStep.java @@ -0,0 +1,108 @@ +package ca.uhn.fhir.batch2.jobs.expunge; + +/*- + * #%L + * hapi-fhir-storage-batch2-jobs + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.batch2.api.IJobDataSink; +import ca.uhn.fhir.batch2.api.IJobStepWorker; +import ca.uhn.fhir.batch2.api.JobExecutionFailedException; +import ca.uhn.fhir.batch2.api.RunOutcome; +import ca.uhn.fhir.batch2.api.StepExecutionDetails; +import ca.uhn.fhir.batch2.api.VoidModel; +import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; +import ca.uhn.fhir.batch2.jobs.reindex.ReindexJobParameters; +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; +import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; +import ca.uhn.fhir.jpa.partition.SystemRequestDetails; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; +import ca.uhn.fhir.util.StopWatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; + +import javax.annotation.Nonnull; +import java.util.List; + +public class DeleteExpungeStep implements IJobStepWorker { + + private static final Logger ourLog = LoggerFactory.getLogger(DeleteExpungeStep.class); + private final HapiTransactionService myHapiTransactionService; + private final IDeleteExpungeSvc myDeleteExpungeSvc; + + public DeleteExpungeStep(HapiTransactionService theHapiTransactionService, IDeleteExpungeSvc theDeleteExpungeSvc) { + myHapiTransactionService = theHapiTransactionService; + myDeleteExpungeSvc = theDeleteExpungeSvc; + } + + @Nonnull + @Override + public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { + + ResourceIdListWorkChunkJson data = theStepExecutionDetails.getData(); + + return doDeleteExpunge(data, theDataSink, theStepExecutionDetails.getInstance().getInstanceId(), theStepExecutionDetails.getChunkId()); + } + + @Nonnull + public RunOutcome doDeleteExpunge(ResourceIdListWorkChunkJson data, IJobDataSink theDataSink, String theInstanceId, String theChunkId) { + RequestDetails requestDetails = new SystemRequestDetails(); + TransactionDetails transactionDetails = new TransactionDetails(); + myHapiTransactionService.execute(requestDetails, transactionDetails, new DeleteExpungeJob(data, requestDetails, transactionDetails, theDataSink, theInstanceId, theChunkId)); + + return new RunOutcome(data.size()); + } + + private class DeleteExpungeJob implements TransactionCallback { + private final ResourceIdListWorkChunkJson myData; + private final RequestDetails myRequestDetails; + private final TransactionDetails myTransactionDetails; + private final IJobDataSink myDataSink; + private final String myChunkId; + private final String myInstanceId; + + public DeleteExpungeJob(ResourceIdListWorkChunkJson theData, RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, IJobDataSink theDataSink, String theInstanceId, String theChunkId) { + myData = theData; + myRequestDetails = theRequestDetails; + myTransactionDetails = theTransactionDetails; + myDataSink = theDataSink; + myInstanceId = theInstanceId; + myChunkId = theChunkId; + } + + @Override + public Void doInTransaction(@Nonnull TransactionStatus theStatus) { + + List persistentIds = myData.getResourcePersistentIds(); + + ourLog.info("Starting delete expunge work chunk with {} resources - Instance[{}] Chunk[{}]", persistentIds.size(), myInstanceId, myChunkId); + StopWatch sw = new StopWatch(); + + myDeleteExpungeSvc.deleteExpunge(persistentIds); + + return null; + } + } + + + +} diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexAppCtx.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexAppCtx.java index b2ea009009e..715902e9002 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexAppCtx.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexAppCtx.java @@ -21,11 +21,17 @@ package ca.uhn.fhir.batch2.jobs.reindex; */ import ca.uhn.fhir.batch2.api.IJobCoordinator; +import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; +import ca.uhn.fhir.batch2.jobs.parameters.UrlListValidator; +import ca.uhn.fhir.batch2.jobs.parameters.UrlPartitioner; +import ca.uhn.fhir.batch2.jobs.step.GenerateRangeChunksStep; +import ca.uhn.fhir.batch2.jobs.step.LoadIdsStep; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; +import ca.uhn.fhir.rest.server.provider.ProviderConstants; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,25 +41,25 @@ public class ReindexAppCtx { public static final String JOB_REINDEX = "REINDEX"; @Bean - public JobDefinition reindexJobDefinition(IResourceReindexSvc theResourceReindexSvc) { + public JobDefinition reindexJobDefinition(IBatch2DaoSvc theBatch2DaoSvc) { return JobDefinition .newBuilder() .setJobDefinitionId(JOB_REINDEX) .setJobDescription("Reindex resources") .setJobDefinitionVersion(1) .setParametersType(ReindexJobParameters.class) - .setParametersValidator(reindexJobParametersValidator()) + .setParametersValidator(reindexJobParametersValidator(theBatch2DaoSvc)) .gatedExecution() .addFirstStep( "generate-ranges", "Generate data ranges to reindex", - ReindexChunkRangeJson.class, + PartitionedUrlChunkRangeJson.class, reindexGenerateRangeChunksStep()) .addIntermediateStep( "load-ids", "Load IDs of resources to reindex", ResourceIdListWorkChunkJson.class, - loadIdsStep(theResourceReindexSvc)) + loadIdsStep(theBatch2DaoSvc)) .addLastStep("reindex", "Perform the resource reindex", reindexStep() @@ -62,8 +68,8 @@ public class ReindexAppCtx { } @Bean - public ReindexJobParametersValidator reindexJobParametersValidator() { - return new ReindexJobParametersValidator(); + public ReindexJobParametersValidator reindexJobParametersValidator(IBatch2DaoSvc theBatch2DaoSvc) { + return new ReindexJobParametersValidator(new UrlListValidator(ProviderConstants.OPERATION_REINDEX, theBatch2DaoSvc)); } @Bean @@ -77,13 +83,13 @@ public class ReindexAppCtx { } @Bean - public LoadIdsStep loadIdsStep(IResourceReindexSvc theResourceReindexSvc) { - return new LoadIdsStep(theResourceReindexSvc); + public LoadIdsStep loadIdsStep(IBatch2DaoSvc theBatch2DaoSvc) { + return new LoadIdsStep(theBatch2DaoSvc); } @Bean - public ReindexProvider reindexProvider(FhirContext theFhirContext, IJobCoordinator theJobCoordinator, IRequestPartitionHelperSvc theRequestPartitionHelperSvc) { - return new ReindexProvider(theFhirContext, theJobCoordinator, theRequestPartitionHelperSvc); + public ReindexProvider reindexProvider(FhirContext theFhirContext, IJobCoordinator theJobCoordinator, IRequestPartitionHelperSvc theRequestPartitionHelperSvc, UrlPartitioner theUrlPartitioner) { + return new ReindexProvider(theFhirContext, theJobCoordinator, theRequestPartitionHelperSvc, theUrlPartitioner); } } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexIdChunkProducer.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexIdChunkProducer.java deleted file mode 100644 index a96355fa060..00000000000 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexIdChunkProducer.java +++ /dev/null @@ -1,49 +0,0 @@ -package ca.uhn.fhir.batch2.jobs.reindex; - -/*- - * #%L - * hapi-fhir-storage-batch2-jobs - * %% - * Copyright (C) 2014 - 2022 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.batch2.jobs.step.IIdChunkProducer; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.pid.IResourcePidList; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.Date; - -public class ReindexIdChunkProducer implements IIdChunkProducer { - private static final Logger ourLog = LoggerFactory.getLogger(ReindexIdChunkProducer.class); - private final IResourceReindexSvc myResourceReindexSvc; - - public ReindexIdChunkProducer(IResourceReindexSvc theResourceReindexSvc) { - myResourceReindexSvc = theResourceReindexSvc; - } - - @Override - public IResourcePidList fetchResourceIdsPage(Date theNextStart, Date theEnd, @Nonnull Integer thePageSize, @Nullable RequestPartitionId theRequestPartitionId, ReindexChunkRangeJson theData) { - String url = theData.getUrl(); - - ourLog.info("Fetching resource ID chunk for URL {} - Range {} - {}", url, theNextStart, theEnd); - return myResourceReindexSvc.fetchResourceIdsPage(theNextStart, theEnd, thePageSize, theRequestPartitionId, url); - } -} diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParameters.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParameters.java index e504d952feb..06c46746b46 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParameters.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParameters.java @@ -20,35 +20,8 @@ package ca.uhn.fhir.batch2.jobs.reindex; * #L% */ -import ca.uhn.fhir.batch2.jobs.parameters.PartitionedJobParameters; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.model.api.IModelJson; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.commons.lang3.Validate; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrlListJobParameters; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import javax.validation.constraints.Pattern; -import java.util.ArrayList; -import java.util.List; - -public class ReindexJobParameters extends PartitionedJobParameters { - - @JsonProperty("url") - @Nullable - private List<@Pattern(regexp = "^[A-Z][A-Za-z0-9]+\\?.*", message = "If populated, URL must be a search URL in the form '{resourceType}?[params]'") String> myUrl; - - public List getUrl() { - if (myUrl == null) { - myUrl = new ArrayList<>(); - } - return myUrl; - } - - public ReindexJobParameters addUrl(@Nonnull String theUrl) { - Validate.notNull(theUrl); - getUrl().add(theUrl); - return this; - } +public class ReindexJobParameters extends PartitionedUrlListJobParameters { } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidator.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidator.java index e2740106d5a..3b475160327 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidator.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidator.java @@ -21,30 +21,22 @@ package ca.uhn.fhir.batch2.jobs.reindex; */ import ca.uhn.fhir.batch2.api.IJobParametersValidator; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; -import ca.uhn.fhir.rest.server.provider.ProviderConstants; -import org.springframework.beans.factory.annotation.Autowired; +import ca.uhn.fhir.batch2.jobs.parameters.UrlListValidator; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.Collections; import java.util.List; public class ReindexJobParametersValidator implements IJobParametersValidator { + private final UrlListValidator myUrlListValidator; - @Autowired - private IResourceReindexSvc myResourceReindexSvc; + public ReindexJobParametersValidator(UrlListValidator theUrlListValidator) { + myUrlListValidator = theUrlListValidator; + } @Nullable @Override - public List validate(@Nonnull ReindexJobParameters theParameters) { - if (theParameters.getUrl().isEmpty()) { - if (!myResourceReindexSvc.isAllResourceTypeSupported()) { - return Collections.singletonList("At least one type-specific search URL must be provided for " + ProviderConstants.OPERATION_REINDEX + " on this server"); - } - } - - return Collections.emptyList(); + public List validate(@NotNull ReindexJobParameters theParameters) { + return myUrlListValidator.validatePartitionedUrls(theParameters.getPartitionedUrls()); } - } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProvider.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProvider.java index 105bc9b1464..67547f57364 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProvider.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProvider.java @@ -21,6 +21,7 @@ package ca.uhn.fhir.batch2.jobs.reindex; */ import ca.uhn.fhir.batch2.api.IJobCoordinator; +import ca.uhn.fhir.batch2.jobs.parameters.UrlPartitioner; import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails; @@ -32,41 +33,42 @@ import ca.uhn.fhir.rest.api.RestOperationTypeEnum; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.util.ParametersUtil; +import org.apache.commons.lang3.StringUtils; import org.hl7.fhir.instance.model.api.IBaseParameters; import org.hl7.fhir.instance.model.api.IPrimitiveType; import java.util.List; -import static org.apache.commons.lang3.StringUtils.isNotBlank; - public class ReindexProvider { private final FhirContext myFhirContext; private final IJobCoordinator myJobCoordinator; private final IRequestPartitionHelperSvc myRequestPartitionHelperSvc; + private final UrlPartitioner myUrlPartitioner; /** * Constructor */ - public ReindexProvider(FhirContext theFhirContext, IJobCoordinator theJobCoordinator, IRequestPartitionHelperSvc theRequestPartitionHelperSvc) { + public ReindexProvider(FhirContext theFhirContext, IJobCoordinator theJobCoordinator, IRequestPartitionHelperSvc theRequestPartitionHelperSvc, UrlPartitioner theUrlPartitioner) { myFhirContext = theFhirContext; myJobCoordinator = theJobCoordinator; myRequestPartitionHelperSvc = theRequestPartitionHelperSvc; + myUrlPartitioner = theUrlPartitioner; } @Operation(name = ProviderConstants.OPERATION_REINDEX, idempotent = false) public IBaseParameters Reindex( - @OperationParam(name = ProviderConstants.OPERATION_REINDEX_PARAM_URL, typeName = "string", min = 0, max = OperationParam.MAX_UNLIMITED) List> theUrl, + @OperationParam(name = ProviderConstants.OPERATION_REINDEX_PARAM_URL, typeName = "string", min = 0, max = OperationParam.MAX_UNLIMITED) List> theUrlsToReindex, RequestDetails theRequestDetails ) { ReindexJobParameters params = new ReindexJobParameters(); - if (theUrl != null) { - theUrl - .stream() - .map(t -> t.getValue()) - .filter(t -> isNotBlank(t)) - .forEach(t -> params.getUrl().add(t)); + if (theUrlsToReindex != null) { + theUrlsToReindex.stream() + .map(IPrimitiveType::getValue) + .filter(StringUtils::isNotBlank) + .map(url -> myUrlPartitioner.partitionUrl(url, theRequestDetails)) + .forEach(params::addPartitionedUrl); } ReadPartitionIdRequestDetails details= new ReadPartitionIdRequestDetails(null, RestOperationTypeEnum.EXTENDED_OPERATION_SERVER, null, null, null); diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/BaseR4ServerTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/BaseR4ServerTest.java new file mode 100644 index 00000000000..0b631445657 --- /dev/null +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/BaseR4ServerTest.java @@ -0,0 +1,45 @@ +package ca.uhn.fhir.batch2.jobs; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.api.BundleInclusionRule; +import ca.uhn.fhir.rest.api.EncodingEnum; +import ca.uhn.fhir.rest.client.api.IGenericClient; +import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; +import ca.uhn.fhir.rest.server.RestfulServer; +import ca.uhn.fhir.test.utilities.JettyUtil; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.jupiter.api.AfterEach; + +public class BaseR4ServerTest { + protected FhirContext myCtx = FhirContext.forR4Cached(); + private Server myServer; + protected IGenericClient myClient; + protected String myBaseUrl; + + @AfterEach + public void after() throws Exception { + JettyUtil.closeServer(myServer); + } + + protected void startServer(Object theProvider) throws Exception { + RestfulServer servlet = new RestfulServer(myCtx); + servlet.registerProvider(theProvider); + ServletHandler proxyHandler = new ServletHandler(); + servlet.setDefaultResponseEncoding(EncodingEnum.XML); + servlet.setBundleInclusionRule(BundleInclusionRule.BASED_ON_RESOURCE_PRESENCE); + ServletHolder servletHolder = new ServletHolder(servlet); + proxyHandler.addServletWithMapping(servletHolder, "/*"); + + myServer = new Server(0); + myServer.setHandler(proxyHandler); + JettyUtil.startServer(myServer); + int port = JettyUtil.getPortForStartedServer(myServer); + + myBaseUrl = "http://localhost:" + port; + myCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER); + myClient = myCtx.newRestfulGenericClient(myBaseUrl); + } + +} diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeProviderTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeProviderTest.java new file mode 100644 index 00000000000..5971b415696 --- /dev/null +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/expunge/DeleteExpungeProviderTest.java @@ -0,0 +1,80 @@ +package ca.uhn.fhir.batch2.jobs.expunge; + +import ca.uhn.fhir.batch2.jobs.BaseR4ServerTest; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; +import ca.uhn.fhir.rest.server.provider.ProviderConstants; +import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; +import org.hl7.fhir.r4.model.DecimalType; +import org.hl7.fhir.r4.model.Parameters; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class DeleteExpungeProviderTest extends BaseR4ServerTest { + public static final String TEST_JOB_ID = "test-job-id"; + private static final Logger ourLog = LoggerFactory.getLogger(DeleteExpungeProviderTest.class); + + private Parameters myReturnParameters; + private MyDeleteExpungeJobSubmitter myDeleteExpungeJobSubmitter = new MyDeleteExpungeJobSubmitter(); + + @BeforeEach + public void reset() { + myReturnParameters = new Parameters(); + myReturnParameters.addParameter("success", true); + } + + @Test + public void testDeleteExpunge() throws Exception { + // setup + Parameters input = new Parameters(); + String url1 = "Observation?status=active"; + String url2 = "Patient?active=false"; + Integer batchSize = 2401; + input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, url1); + input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, url2); + input.addParameter(ProviderConstants.OPERATION_DELETE_BATCH_SIZE, new DecimalType(batchSize)); + + ourLog.info(myCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(input)); + + DeleteExpungeProvider provider = new DeleteExpungeProvider(myCtx, myDeleteExpungeJobSubmitter); + startServer(provider); + + Parameters response = myClient + .operation() + .onServer() + .named(ProviderConstants.OPERATION_DELETE_EXPUNGE) + .withParameters(input) + .execute(); + + ourLog.info(myCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(response)); + assertEquals(TEST_JOB_ID, BatchHelperR4.jobIdFromBatch2Parameters(response)); + assertThat(myDeleteExpungeJobSubmitter.calledWithUrls, hasSize(2)); + assertEquals(url1, myDeleteExpungeJobSubmitter.calledWithUrls.get(0)); + assertEquals(url2, myDeleteExpungeJobSubmitter.calledWithUrls.get(1)); + assertEquals(batchSize, myDeleteExpungeJobSubmitter.calledWithBatchSize); + assertNotNull(myDeleteExpungeJobSubmitter.calledWithRequestDetails); + } + + private class MyDeleteExpungeJobSubmitter implements IDeleteExpungeJobSubmitter { + Integer calledWithBatchSize; + List calledWithUrls; + RequestDetails calledWithRequestDetails; + + @Override + public String submitJob(Integer theBatchSize, List theUrlsToProcess, RequestDetails theRequest) { + calledWithBatchSize = theBatchSize; + calledWithUrls = theUrlsToProcess; + calledWithRequestDetails = theRequest; + return TEST_JOB_ID; + } + } +} diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidatorTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidatorTest.java deleted file mode 100644 index a6d5dcc97b5..00000000000 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexJobParametersValidatorTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package ca.uhn.fhir.batch2.jobs.reindex; - -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; -import org.hamcrest.Matchers; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.empty; -import static org.mockito.Mockito.when; - - -@ExtendWith(MockitoExtension.class) -public class ReindexJobParametersValidatorTest { - - @Mock - private IResourceReindexSvc myResourceReindexSvc; - - @InjectMocks - private ReindexJobParametersValidator mySvc; - - @Test - public void testAllResourceTypeSupportedTrue() { - when(myResourceReindexSvc.isAllResourceTypeSupported()).thenReturn(true); - - assertThat(mySvc.validate(new ReindexJobParameters()), empty()); - assertThat(mySvc.validate(new ReindexJobParameters().addUrl("Patient?")), empty()); - } - - @Test - public void testAllResourceTypeSupportedFalse() { - when(myResourceReindexSvc.isAllResourceTypeSupported()).thenReturn(false); - - assertThat(mySvc.validate(new ReindexJobParameters()), Matchers.contains("At least one type-specific search URL must be provided for $reindex on this server")); - assertThat(mySvc.validate(new ReindexJobParameters().addUrl("Patient?")), empty()); - } - -} diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProviderTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProviderTest.java index 4f8e23c159b..53f4af8307d 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProviderTest.java +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexProviderTest.java @@ -1,13 +1,14 @@ package ca.uhn.fhir.batch2.jobs.reindex; import ca.uhn.fhir.batch2.api.IJobCoordinator; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrl; +import ca.uhn.fhir.batch2.jobs.parameters.UrlPartitioner; import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.test.utilities.server.RestfulServerExtension; -import org.hamcrest.Matchers; import org.hl7.fhir.r4.model.BooleanType; import org.hl7.fhir.r4.model.DecimalType; import org.hl7.fhir.r4.model.Parameters; @@ -28,8 +29,10 @@ import org.slf4j.LoggerFactory; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -50,6 +53,8 @@ public class ReindexProviderTest { @Mock private IRequestPartitionHelperSvc myRequestPartitionHelperSvc; + @Mock + private UrlPartitioner myUrlPartitioner; @Captor private ArgumentCaptor myStartRequestCaptor; @@ -81,6 +86,11 @@ public class ReindexProviderTest { ourLog.info(myCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(input)); + PartitionedUrl partitionedUrl = new PartitionedUrl(); + partitionedUrl.setUrl(url); + partitionedUrl.setRequestPartitionId(RequestPartitionId.defaultPartition()); + when(myUrlPartitioner.partitionUrl(anyString(), any())).thenReturn(partitionedUrl); + // Execute Parameters response = myServerExtension @@ -99,7 +109,8 @@ public class ReindexProviderTest { verify(myJobCoordinator, times(1)).startInstance(myStartRequestCaptor.capture()); ReindexJobParameters params = myStartRequestCaptor.getValue().getParameters(ReindexJobParameters.class); - assertThat(params.getUrl(), Matchers.contains(url)); + assertThat(params.getPartitionedUrls(), hasSize(1)); + assertEquals(url, params.getPartitionedUrls().get(0).getUrl()); } @Test @@ -130,7 +141,7 @@ public class ReindexProviderTest { verify(myJobCoordinator, times(1)).startInstance(myStartRequestCaptor.capture()); ReindexJobParameters params = myStartRequestCaptor.getValue().getParameters(ReindexJobParameters.class); - assertThat(params.getUrl(), empty()); + assertThat(params.getPartitionedUrls(), empty()); } } diff --git a/hapi-fhir-storage-batch2/pom.xml b/hapi-fhir-storage-batch2/pom.xml index 8e288cc7e3e..e865c05a347 100644 --- a/hapi-fhir-storage-batch2/pom.xml +++ b/hapi-fhir-storage-batch2/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java index 625e5ecd8b6..8c561a3f0fc 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java @@ -30,12 +30,16 @@ import ca.uhn.fhir.batch2.model.WorkChunkData; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.JsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; class JobDataSink extends BaseDataSink { + private static final Logger ourLog = LoggerFactory.getLogger(JobDataSink.class); + private final BatchJobSender myBatchJobSender; private final IJobPersistence myJobPersistence; private final String myJobDefinitionId; diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexChunkRangeJson.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/PartitionedUrlChunkRangeJson.java similarity index 61% rename from hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexChunkRangeJson.java rename to hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/PartitionedUrlChunkRangeJson.java index ab0c38905a3..1bbe50b3f89 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/ReindexChunkRangeJson.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/chunk/PartitionedUrlChunkRangeJson.java @@ -1,8 +1,8 @@ -package ca.uhn.fhir.batch2.jobs.reindex; +package ca.uhn.fhir.batch2.jobs.chunk; /*- * #%L - * hapi-fhir-storage-batch2-jobs + * HAPI FHIR JPA Server - Batch2 Task Processor * %% * Copyright (C) 2014 - 2022 Smile CDR, Inc. * %% @@ -20,22 +20,22 @@ package ca.uhn.fhir.batch2.jobs.reindex; * #L% */ -import ca.uhn.fhir.batch2.jobs.chunk.ChunkRangeJson; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrl; import com.fasterxml.jackson.annotation.JsonProperty; import javax.annotation.Nullable; -public class ReindexChunkRangeJson extends ChunkRangeJson { +public class PartitionedUrlChunkRangeJson extends ChunkRangeJson { @Nullable - @JsonProperty("url") - private String myUrl; + @JsonProperty("partitionedUrl") + private PartitionedUrl myPartitionedUrl; @Nullable - public String getUrl() { - return myUrl; + public PartitionedUrl getPartitionedUrl() { + return myPartitionedUrl; } - public void setUrl(@Nullable String theUrl) { - myUrl = theUrl; + public void setPartitionedUrl(@Nullable PartitionedUrl thePartitionedUrl) { + myPartitionedUrl = thePartitionedUrl; } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrl.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrl.java new file mode 100644 index 00000000000..1805afe6a40 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrl.java @@ -0,0 +1,30 @@ +package ca.uhn.fhir.batch2.jobs.parameters; + +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.model.api.IModelJson; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Pattern; + +public class PartitionedUrl implements IModelJson { + @JsonProperty("url") + @Pattern(regexp = "^[A-Z][A-Za-z0-9]+\\?.*", message = "If populated, URL must be a search URL in the form '{resourceType}?[params]'") String myUrl; + @JsonProperty("requestPartitionId") + RequestPartitionId myRequestPartitionId; + + public String getUrl() { + return myUrl; + } + + public void setUrl(String theUrl) { + myUrl = theUrl; + } + + public RequestPartitionId getRequestPartitionId() { + return myRequestPartitionId; + } + + public void setRequestPartitionId(RequestPartitionId theRequestPartitionId) { + myRequestPartitionId = theRequestPartitionId; + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrlListJobParameters.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrlListJobParameters.java new file mode 100644 index 00000000000..e099b2120f1 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/PartitionedUrlListJobParameters.java @@ -0,0 +1,56 @@ +package ca.uhn.fhir.batch2.jobs.parameters; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang3.Validate; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; + +public class PartitionedUrlListJobParameters extends PartitionedJobParameters { + @JsonProperty("partitionedUrl") + @Nullable + private List myPartitionedUrls; + + public List getPartitionedUrls() { + if (myPartitionedUrls == null) { + myPartitionedUrls = new ArrayList<>(); + } + return myPartitionedUrls; + } + + public PartitionedUrlListJobParameters addPartitionedUrl(@Nonnull PartitionedUrl thePartitionedUrl) { + Validate.notNull(thePartitionedUrl); + getPartitionedUrls().add(thePartitionedUrl); + return this; + } + + public PartitionedUrlListJobParameters addUrl(@Nonnull String theUrl) { + PartitionedUrl partitionedUrl = new PartitionedUrl(); + partitionedUrl.setUrl(theUrl); + partitionedUrl.setRequestPartitionId(RequestPartitionId.defaultPartition()); + return addPartitionedUrl(partitionedUrl); + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListValidator.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListValidator.java new file mode 100644 index 00000000000..546407a5c66 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListValidator.java @@ -0,0 +1,56 @@ +package ca.uhn.fhir.batch2.jobs.parameters; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class UrlListValidator { + private final String myOperationName; + private final IBatch2DaoSvc myBatch2DaoSvc; + + public UrlListValidator(String theOperationName, IBatch2DaoSvc theBatch2DaoSvc) { + myOperationName = theOperationName; + myBatch2DaoSvc = theBatch2DaoSvc; + } + + + @Nullable + public List validateUrls(@Nonnull List theUrls) { + if (theUrls.isEmpty()) { + if (!myBatch2DaoSvc.isAllResourceTypeSupported()) { + return Collections.singletonList("At least one type-specific search URL must be provided for " + myOperationName + " on this server"); + } + } + return Collections.emptyList(); + } + + @Nullable + public List validatePartitionedUrls(@Nonnull List thePartitionedUrls) { + List urls = thePartitionedUrls.stream().map(PartitionedUrl::getUrl).collect(Collectors.toList()); + return validateUrls(urls); + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlPartitioner.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlPartitioner.java new file mode 100644 index 00000000000..06cd5b20744 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/parameters/UrlPartitioner.java @@ -0,0 +1,27 @@ +package ca.uhn.fhir.batch2.jobs.parameters; + +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; +import ca.uhn.fhir.jpa.searchparam.MatchUrlService; +import ca.uhn.fhir.jpa.searchparam.ResourceSearch; +import ca.uhn.fhir.rest.api.server.RequestDetails; + +public class UrlPartitioner { + private final MatchUrlService myMatchUrlService; + private final IRequestPartitionHelperSvc myRequestPartitionHelperSvc; + + public UrlPartitioner(MatchUrlService theMatchUrlService, IRequestPartitionHelperSvc theRequestPartitionHelperSvc) { + myMatchUrlService = theMatchUrlService; + myRequestPartitionHelperSvc = theRequestPartitionHelperSvc; + + } + + public PartitionedUrl partitionUrl(String theUrl, RequestDetails theRequestDetails) { + ResourceSearch resourceSearch = myMatchUrlService.getResourceSearch(theUrl); + RequestPartitionId requestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequestForSearchType(theRequestDetails, resourceSearch.getResourceName(), resourceSearch.getSearchParameterMap(), null); + PartitionedUrl retval = new PartitionedUrl(); + retval.setUrl(theUrl); + retval.setRequestPartitionId(requestPartitionId); + return retval; + } +} diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/GenerateRangeChunksStep.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/GenerateRangeChunksStep.java similarity index 61% rename from hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/GenerateRangeChunksStep.java rename to hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/GenerateRangeChunksStep.java index 4b8219571b9..0e748801364 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/GenerateRangeChunksStep.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/GenerateRangeChunksStep.java @@ -1,8 +1,8 @@ -package ca.uhn.fhir.batch2.jobs.reindex; +package ca.uhn.fhir.batch2.jobs.step; /*- * #%L - * hapi-fhir-storage-batch2-jobs + * HAPI FHIR JPA Server - Batch2 Task Processor * %% * Copyright (C) 2014 - 2022 Smile CDR, Inc. * %% @@ -26,6 +26,9 @@ import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.api.VoidModel; +import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrl; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrlListJobParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,28 +37,28 @@ import java.util.Date; import static ca.uhn.fhir.batch2.config.Batch2Constants.BATCH_START_DATE; -public class GenerateRangeChunksStep implements IFirstJobStepWorker { +public class GenerateRangeChunksStep implements IFirstJobStepWorker { private static final Logger ourLog = LoggerFactory.getLogger(GenerateRangeChunksStep.class); @Nonnull @Override - public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { - ReindexJobParameters params = theStepExecutionDetails.getParameters(); + public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { + PT params = theStepExecutionDetails.getParameters(); Date start = BATCH_START_DATE; Date end = new Date(); - if (params.getUrl().isEmpty()) { + if (params.getPartitionedUrls().isEmpty()) { ourLog.info("Initiating reindex of All Resources from {} to {}", start, end); - ReindexChunkRangeJson nextRange = new ReindexChunkRangeJson(); + PartitionedUrlChunkRangeJson nextRange = new PartitionedUrlChunkRangeJson(); nextRange.setStart(start); nextRange.setEnd(end); theDataSink.accept(nextRange); } else { - for (String nextUrl : params.getUrl()) { - ourLog.info("Initiating reindex of [{}]] from {} to {}", nextUrl, start, end); - ReindexChunkRangeJson nextRange = new ReindexChunkRangeJson(); - nextRange.setUrl(nextUrl); + for (PartitionedUrl nextPartitionedUrl : params.getPartitionedUrls()) { + ourLog.info("Initiating reindex of [{}]] from {} to {}", nextPartitionedUrl, start, end); + PartitionedUrlChunkRangeJson nextRange = new PartitionedUrlChunkRangeJson(); + nextRange.setPartitionedUrl(nextPartitionedUrl); nextRange.setStart(start); nextRange.setEnd(end); theDataSink.accept(nextRange); diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/LoadIdsStep.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStep.java similarity index 52% rename from hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/LoadIdsStep.java rename to hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStep.java index 1522cd812bd..cd2594710bb 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/reindex/LoadIdsStep.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStep.java @@ -1,8 +1,8 @@ -package ca.uhn.fhir.batch2.jobs.reindex; +package ca.uhn.fhir.batch2.jobs.step; /*- * #%L - * hapi-fhir-storage-batch2-jobs + * HAPI FHIR JPA Server - Batch2 Task Processor * %% * Copyright (C) 2014 - 2022 Smile CDR, Inc. * %% @@ -25,28 +25,28 @@ import ca.uhn.fhir.batch2.api.IJobStepWorker; import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; +import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; -import ca.uhn.fhir.batch2.jobs.step.IIdChunkProducer; -import ca.uhn.fhir.batch2.jobs.step.ResourceIdListStep; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrlListJobParameters; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; import javax.annotation.Nonnull; -public class LoadIdsStep implements IJobStepWorker { - private final IResourceReindexSvc myResourceReindexSvc; +public class LoadIdsStep implements IJobStepWorker { + private final IBatch2DaoSvc myBatch2DaoSvc; - private final ResourceIdListStep myResourceIdListStep; + private final ResourceIdListStep myResourceIdListStep; - public LoadIdsStep(IResourceReindexSvc theResourceReindexSvc) { - myResourceReindexSvc = theResourceReindexSvc; + public LoadIdsStep(IBatch2DaoSvc theBatch2DaoSvc) { + myBatch2DaoSvc = theBatch2DaoSvc; - IIdChunkProducer idChunkProducer = new ReindexIdChunkProducer(theResourceReindexSvc); + IIdChunkProducer idChunkProducer = new PartitionedUrlListIdChunkProducer(theBatch2DaoSvc); myResourceIdListStep = new ResourceIdListStep<>(idChunkProducer); } @Nonnull @Override - public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { + public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, @Nonnull IJobDataSink theDataSink) throws JobExecutionFailedException { return myResourceIdListStep.run(theStepExecutionDetails, theDataSink); } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/PartitionedUrlListIdChunkProducer.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/PartitionedUrlListIdChunkProducer.java new file mode 100644 index 00000000000..e7e956d036b --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/jobs/step/PartitionedUrlListIdChunkProducer.java @@ -0,0 +1,55 @@ +package ca.uhn.fhir.batch2.jobs.step; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2022 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrl; +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.api.pid.IResourcePidList; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Date; + +public class PartitionedUrlListIdChunkProducer implements IIdChunkProducer { + private static final Logger ourLog = LoggerFactory.getLogger(PartitionedUrlListIdChunkProducer.class); + private final IBatch2DaoSvc myBatch2DaoSvc; + + public PartitionedUrlListIdChunkProducer(IBatch2DaoSvc theBatch2DaoSvc) { + myBatch2DaoSvc = theBatch2DaoSvc; + } + + @Override + public IResourcePidList fetchResourceIdsPage(Date theNextStart, Date theEnd, @Nonnull Integer thePageSize, @Nullable RequestPartitionId theRequestPartitionId, PartitionedUrlChunkRangeJson theData) { + PartitionedUrl partitionedUrl = theData.getPartitionedUrl(); + + if (partitionedUrl == null) { + ourLog.info("Fetching resource ID chunk for everything - Range {} - {}", theNextStart, theEnd); + return myBatch2DaoSvc.fetchResourceIdsPage(theNextStart, theEnd, thePageSize, theRequestPartitionId, null); + } else { + ourLog.info("Fetching resource ID chunk for URL {} - Range {} - {}", partitionedUrl.getUrl(), theNextStart, theEnd); + return myBatch2DaoSvc.fetchResourceIdsPage(theNextStart, theEnd, thePageSize, partitionedUrl.getRequestPartitionId(), partitionedUrl.getUrl()); + } + } +} diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListParametersValidatorTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListParametersValidatorTest.java new file mode 100644 index 00000000000..24674aab311 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/parameters/UrlListParametersValidatorTest.java @@ -0,0 +1,48 @@ +package ca.uhn.fhir.batch2.jobs.parameters; + +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.mockito.Mockito.when; + + +@ExtendWith(MockitoExtension.class) +public class UrlListParametersValidatorTest { + + @Mock + private IBatch2DaoSvc myBatch2DaoSvc; + + private UrlListValidator mySvc; + + @BeforeEach + public void before() { + mySvc = new UrlListValidator("TESTOP", myBatch2DaoSvc); + } + + @Test + public void testAllResourceTypeSupportedTrue() { + when(myBatch2DaoSvc.isAllResourceTypeSupported()).thenReturn(true); + + assertThat(mySvc.validateUrls(Collections.emptyList()), empty()); + assertThat(mySvc.validateUrls(List.of("Patient?")), empty()); + } + + @Test + public void testAllResourceTypeSupportedFalse() { + when(myBatch2DaoSvc.isAllResourceTypeSupported()).thenReturn(false); + + assertThat(mySvc.validateUrls(Collections.emptyList()), Matchers.contains("At least one type-specific search URL must be provided for TESTOP on this server")); + assertThat(mySvc.validateUrls(List.of("Patient?")), empty()); + } + +} diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/LoadIdsStepTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStepTest.java similarity index 77% rename from hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/LoadIdsStepTest.java rename to hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStepTest.java index 213d999ea3f..ecfd198cfa8 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/reindex/LoadIdsStepTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/jobs/step/LoadIdsStepTest.java @@ -1,12 +1,14 @@ -package ca.uhn.fhir.batch2.jobs.reindex; +package ca.uhn.fhir.batch2.jobs.step; import ca.uhn.fhir.batch2.api.IJobDataSink; import ca.uhn.fhir.batch2.api.StepExecutionDetails; +import ca.uhn.fhir.batch2.jobs.chunk.PartitionedUrlChunkRangeJson; import ca.uhn.fhir.batch2.jobs.chunk.ResourceIdListWorkChunkJson; +import ca.uhn.fhir.batch2.jobs.parameters.PartitionedUrlListJobParameters; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.jpa.api.pid.HomogeneousResourcePidList; import ca.uhn.fhir.jpa.api.pid.IResourcePidList; -import ca.uhn.fhir.jpa.api.svc.IResourceReindexSvc; +import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc; import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; import org.hl7.fhir.r4.model.InstantType; import org.junit.jupiter.api.BeforeEach; @@ -40,7 +42,7 @@ public class LoadIdsStepTest { public static final Date DATE_END = new InstantType("2022-02-01T00:00:00Z").getValue(); @Mock - private IResourceReindexSvc myResourceReindexSvc; + private IBatch2DaoSvc myBatch2DaoSvc; @Mock private IJobDataSink mySink; @@ -49,7 +51,7 @@ public class LoadIdsStepTest { @BeforeEach public void before() { - mySvc = new LoadIdsStep(myResourceReindexSvc); + mySvc = new LoadIdsStep(myBatch2DaoSvc); } @Captor @@ -57,21 +59,21 @@ public class LoadIdsStepTest { @Test public void testGenerateSteps() { - ReindexJobParameters parameters = new ReindexJobParameters(); - ReindexChunkRangeJson range = new ReindexChunkRangeJson(); + PartitionedUrlListJobParameters parameters = new PartitionedUrlListJobParameters(); + PartitionedUrlChunkRangeJson range = new PartitionedUrlChunkRangeJson(); range.setStart(DATE_1).setEnd(DATE_END); String instanceId = "instance-id"; JobInstance jobInstance = JobInstance.fromInstanceId(instanceId); String chunkId = "chunk-id"; - StepExecutionDetails details = new StepExecutionDetails<>(parameters, range, jobInstance, chunkId); + StepExecutionDetails details = new StepExecutionDetails<>(parameters, range, jobInstance, chunkId); // First Execution - when(myResourceReindexSvc.fetchResourceIdsPage(eq(DATE_1), eq(DATE_END), eq(DEFAULT_PAGE_SIZE), isNull(), isNull())) + when(myBatch2DaoSvc.fetchResourceIdsPage(eq(DATE_1), eq(DATE_END), eq(DEFAULT_PAGE_SIZE), isNull(), isNull())) .thenReturn(createIdChunk(0L, 20000L, DATE_2)); - when(myResourceReindexSvc.fetchResourceIdsPage(eq(DATE_2), eq(DATE_END), eq(DEFAULT_PAGE_SIZE), isNull(), isNull())) + when(myBatch2DaoSvc.fetchResourceIdsPage(eq(DATE_2), eq(DATE_END), eq(DEFAULT_PAGE_SIZE), isNull(), isNull())) .thenReturn(createIdChunk(20000L, 40000L, DATE_3)); - when(myResourceReindexSvc.fetchResourceIdsPage(eq(DATE_3), eq(DATE_END), eq(DEFAULT_PAGE_SIZE), isNull(), isNull())) + when(myBatch2DaoSvc.fetchResourceIdsPage(eq(DATE_3), eq(DATE_END), eq(DEFAULT_PAGE_SIZE), isNull(), isNull())) .thenReturn(createIdChunk(40000L, 40040L, DATE_3)); mySvc.run(details, mySink); diff --git a/hapi-fhir-storage-mdm/pom.xml b/hapi-fhir-storage-mdm/pom.xml index 4c19fcbc852..25bb56b3b29 100644 --- a/hapi-fhir-storage-mdm/pom.xml +++ b/hapi-fhir-storage-mdm/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-storage-test-utilities/pom.xml b/hapi-fhir-storage-test-utilities/pom.xml index a1bd83d8bb8..2a681bc43d6 100644 --- a/hapi-fhir-storage-test-utilities/pom.xml +++ b/hapi-fhir-storage-test-utilities/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-storage/pom.xml b/hapi-fhir-storage/pom.xml index aa95daca34e..a4a8d2f89aa 100644 --- a/hapi-fhir-storage/pom.xml +++ b/hapi-fhir-storage/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IResourceReindexSvc.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IBatch2DaoSvc.java similarity index 98% rename from hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IResourceReindexSvc.java rename to hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IBatch2DaoSvc.java index 1ec7d720b27..def3eb54805 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IResourceReindexSvc.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IBatch2DaoSvc.java @@ -27,7 +27,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Date; -public interface IResourceReindexSvc { +public interface IBatch2DaoSvc { /** * Indicates whether reindexing all resource types is supported. Implementations are expected to provide a static response (either they support this or they don't). diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmClearJobSubmitter.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IDeleteExpungeSvc.java similarity index 59% rename from hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmClearJobSubmitter.java rename to hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IDeleteExpungeSvc.java index 311ab23c84c..7dce4d424b3 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmClearJobSubmitter.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IDeleteExpungeSvc.java @@ -1,8 +1,8 @@ -package ca.uhn.fhir.mdm.api; +package ca.uhn.fhir.jpa.api.svc; /*- * #%L - * HAPI FHIR - Master Data Management + * HAPI FHIR Storage api * %% * Copyright (C) 2014 - 2022 Smile CDR, Inc. * %% @@ -20,10 +20,14 @@ package ca.uhn.fhir.mdm.api; * #L% */ -import ca.uhn.fhir.rest.api.server.storage.IMultiUrlJobSubmitter; +import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; -/** - * Tag interface for Spring autowiring - */ -public interface IMdmClearJobSubmitter extends IMultiUrlJobSubmitter { +import java.util.List; + +@Transactional(propagation = Propagation.MANDATORY) +public interface IDeleteExpungeSvc { + + void deleteExpunge(List thePersistentIds); } diff --git a/hapi-fhir-structures-dstu2.1/pom.xml b/hapi-fhir-structures-dstu2.1/pom.xml index 431ac1b9ae1..173b407d472 100644 --- a/hapi-fhir-structures-dstu2.1/pom.xml +++ b/hapi-fhir-structures-dstu2.1/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu2/pom.xml b/hapi-fhir-structures-dstu2/pom.xml index 6179432fdb6..0e1f3497252 100644 --- a/hapi-fhir-structures-dstu2/pom.xml +++ b/hapi-fhir-structures-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu3/pom.xml b/hapi-fhir-structures-dstu3/pom.xml index 96f27edb1e7..18ec2ba3cb7 100644 --- a/hapi-fhir-structures-dstu3/pom.xml +++ b/hapi-fhir-structures-dstu3/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-hl7org-dstu2/pom.xml b/hapi-fhir-structures-hl7org-dstu2/pom.xml index 8cb9824d969..d9e39312229 100644 --- a/hapi-fhir-structures-hl7org-dstu2/pom.xml +++ b/hapi-fhir-structures-hl7org-dstu2/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4/pom.xml b/hapi-fhir-structures-r4/pom.xml index 09fccf64480..48d5ca578f5 100644 --- a/hapi-fhir-structures-r4/pom.xml +++ b/hapi-fhir-structures-r4/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java b/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java deleted file mode 100644 index d5d94767a33..00000000000 --- a/hapi-fhir-structures-r4/src/test/java/ca/uhn/fhir/rest/server/provider/BatchProviderTest.java +++ /dev/null @@ -1,116 +0,0 @@ -package ca.uhn.fhir.rest.server.provider; - -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter; -import ca.uhn.fhir.rest.api.server.storage.IReindexJobSubmitter; -import ca.uhn.fhir.rest.server.BaseR4ServerTest; -import org.hl7.fhir.r4.hapi.rest.server.helper.BatchHelperR4; -import org.hl7.fhir.r4.model.DecimalType; -import org.hl7.fhir.r4.model.Parameters; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersInvalidException; - -import javax.annotation.Nonnull; -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasSize; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -public class BatchProviderTest extends BaseR4ServerTest { - public static final long TEST_JOB_ID = 123L; - public static final String TEST_JOB_NAME = "jobName"; - private static final Logger ourLog = LoggerFactory.getLogger(BatchProviderTest.class); - private final MyMultiUrlJobSubmitter myDeleteExpungeJobSubmitter = new MyMultiUrlJobSubmitter(ProviderConstants.OPERATION_DELETE_EXPUNGE); - private final MyMultiUrlJobSubmitter myReindexJobSubmitter = new MyMultiUrlJobSubmitter(ProviderConstants.OPERATION_REINDEX); - private Parameters myReturnParameters; - - @BeforeEach - public void reset() { - myReturnParameters = new Parameters(); - myReturnParameters.addParameter("success", true); - myDeleteExpungeJobSubmitter.reset(); - myReindexJobSubmitter.reset(); - } - - @Test - public void testDeleteExpunge() throws Exception { - // setup - Parameters input = new Parameters(); - String url1 = "Observation?status=active"; - String url2 = "Patient?active=false"; - Integer batchSize = 2401; - input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, url1); - input.addParameter(ProviderConstants.OPERATION_DELETE_EXPUNGE_URL, url2); - input.addParameter(ProviderConstants.OPERATION_DELETE_BATCH_SIZE, new DecimalType(batchSize)); - - ourLog.info(myCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(input)); - - DeleteExpungeProvider provider = new DeleteExpungeProvider(myCtx, myDeleteExpungeJobSubmitter); - startServer(provider); - - Parameters response = myClient - .operation() - .onServer() - .named(ProviderConstants.OPERATION_DELETE_EXPUNGE) - .withParameters(input) - .execute(); - - ourLog.info(myCtx.newJsonParser().setPrettyPrint(true).encodeResourceToString(response)); - assertEquals(TEST_JOB_ID, BatchHelperR4.jobIdFromParameters(response)); - assertThat(myDeleteExpungeJobSubmitter.calledWithUrls, hasSize(2)); - assertEquals(url1, myDeleteExpungeJobSubmitter.calledWithUrls.get(0)); - assertEquals(url2, myDeleteExpungeJobSubmitter.calledWithUrls.get(1)); - assertEquals(batchSize, myDeleteExpungeJobSubmitter.calledWithBatchSize); - assertNotNull(myDeleteExpungeJobSubmitter.calledWithRequestDetails); - assertFalse(myDeleteExpungeJobSubmitter.everything); - } - - private class MyMultiUrlJobSubmitter implements IReindexJobSubmitter, IDeleteExpungeJobSubmitter { - public final String operationName; - public Integer calledWithBatchSize; - public RequestDetails calledWithRequestDetails; - public List calledWithUrls; - public boolean everything; - - public MyMultiUrlJobSubmitter(String theOperationName) { - operationName = theOperationName; - } - - @Override - public JobExecution submitJob(Integer theBatchSize, List theUrlsToProcess, RequestDetails theRequestDetails) { - calledWithBatchSize = theBatchSize; - calledWithRequestDetails = theRequestDetails; - calledWithUrls = theUrlsToProcess; - everything = false; - return buildJobExecution(); - } - - @Nonnull - private JobExecution buildJobExecution() { - JobInstance instance = new JobInstance(TEST_JOB_ID, TEST_JOB_NAME); - return new JobExecution(instance, new JobParameters()); - } - - public void reset() { - calledWithUrls = new ArrayList<>(); - } - - @Override - public JobExecution submitEverythingJob(Integer theBatchSize, RequestDetails theRequestDetails) throws JobParametersInvalidException { - calledWithBatchSize = theBatchSize; - calledWithRequestDetails = theRequestDetails; - everything = true; - return buildJobExecution(); - } - } -} diff --git a/hapi-fhir-structures-r5/pom.xml b/hapi-fhir-structures-r5/pom.xml index ec73bd3200a..d4ea94a37e7 100644 --- a/hapi-fhir-structures-r5/pom.xml +++ b/hapi-fhir-structures-r5/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-test-utilities/pom.xml b/hapi-fhir-test-utilities/pom.xml index c562ec2b7dc..57b46c57d7c 100644 --- a/hapi-fhir-test-utilities/pom.xml +++ b/hapi-fhir-test-utilities/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-testpage-overlay/pom.xml b/hapi-fhir-testpage-overlay/pom.xml index c04e70fda34..b0302f3663e 100644 --- a/hapi-fhir-testpage-overlay/pom.xml +++ b/hapi-fhir-testpage-overlay/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-validation-resources-dstu2.1/pom.xml b/hapi-fhir-validation-resources-dstu2.1/pom.xml index 2d0e8f05505..c7288aa3457 100644 --- a/hapi-fhir-validation-resources-dstu2.1/pom.xml +++ b/hapi-fhir-validation-resources-dstu2.1/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu2/pom.xml b/hapi-fhir-validation-resources-dstu2/pom.xml index 600970ecac7..2681a62b167 100644 --- a/hapi-fhir-validation-resources-dstu2/pom.xml +++ b/hapi-fhir-validation-resources-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu3/pom.xml b/hapi-fhir-validation-resources-dstu3/pom.xml index be42fb8f35a..072543fd580 100644 --- a/hapi-fhir-validation-resources-dstu3/pom.xml +++ b/hapi-fhir-validation-resources-dstu3/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r4/pom.xml b/hapi-fhir-validation-resources-r4/pom.xml index b988f351eef..2f651e7f070 100644 --- a/hapi-fhir-validation-resources-r4/pom.xml +++ b/hapi-fhir-validation-resources-r4/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r5/pom.xml b/hapi-fhir-validation-resources-r5/pom.xml index 7fb3f309e83..367a15baaed 100644 --- a/hapi-fhir-validation-resources-r5/pom.xml +++ b/hapi-fhir-validation-resources-r5/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation/pom.xml b/hapi-fhir-validation/pom.xml index 7eff347e8fb..3d4785f729c 100644 --- a/hapi-fhir-validation/pom.xml +++ b/hapi-fhir-validation/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-tinder-plugin/pom.xml b/hapi-tinder-plugin/pom.xml index 328dae44b62..d848a207df7 100644 --- a/hapi-tinder-plugin/pom.xml +++ b/hapi-tinder-plugin/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml @@ -58,37 +58,37 @@ ca.uhn.hapi.fhir hapi-fhir-structures-dstu3 - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-structures-hl7org-dstu2 - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-structures-r4 - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-structures-r5 - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-validation-resources-dstu2 - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-validation-resources-dstu3 - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ca.uhn.hapi.fhir hapi-fhir-validation-resources-r4 - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT org.apache.velocity diff --git a/hapi-tinder-test/pom.xml b/hapi-tinder-test/pom.xml index 2ae443f68e7..27fffc92642 100644 --- a/hapi-tinder-test/pom.xml +++ b/hapi-tinder-test/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 4f79b5b8cd1..1a813ac79da 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir pom - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT HAPI-FHIR An open-source implementation of the FHIR specification in Java. https://hapifhir.io @@ -2011,7 +2011,7 @@ ca.uhn.hapi.fhir hapi-fhir-checkstyle - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT diff --git a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml index dbc7b4ea802..6ba150b3cd7 100644 --- a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml +++ b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-client/pom.xml b/tests/hapi-fhir-base-test-mindeps-client/pom.xml index 7ae5c0c97e1..38237568c13 100644 --- a/tests/hapi-fhir-base-test-mindeps-client/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-server/pom.xml b/tests/hapi-fhir-base-test-mindeps-server/pom.xml index e0ad30b8d40..461d9ec8c41 100644 --- a/tests/hapi-fhir-base-test-mindeps-server/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.1.0-PRE9-SNAPSHOT + 6.1.0-PRE12-SNAPSHOT ../../pom.xml