From 9754a72046c4a8aca385b3324c7431f4b52c7f49 Mon Sep 17 00:00:00 2001 From: Nathan Doef Date: Fri, 10 Feb 2023 13:14:29 -0500 Subject: [PATCH] Bulk Export Bug With Many Resources and Low Max File Size (#4506) * failing test * fix + changelog * tweak * add method to IJobPersistence to use a Stream * tweak * tweak * decrease test time * clean up * code review comments * version bump * Increase timeout limit to match BulkExportUseCaseTest * shorten test * maintenance pass * add logging * Revert "add logging" This reverts commit b0453fd9538d6858e6bd212f402d4b04c6306136. * Revert "maintenance pass" This reverts commit bbc7418d519260c4b9827546a0bf822db1c78b1b. * test * trying to fix BulkDataExportTest testGroupBulkExportNotInGroup_DoesNotShowUp * shorten tests * logging * move test location * fixes a regression caused my change in hapi-fhir * timeout * Revert "fixes a regression caused my change in hapi-fhir" This reverts commit 4b58013149836eededc568d295c5baf8fb3df989. * testing * Revert "testing" This reverts commit aafc95c2f36bf89ae935c6c342017ec300fe4289. --------- Co-authored-by: leif stawnyczy --- hapi-deployable-pom/pom.xml | 2 +- hapi-fhir-android/pom.xml | 2 +- hapi-fhir-base/pom.xml | 2 +- hapi-fhir-bom/pom.xml | 4 +- hapi-fhir-checkstyle/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-api/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-app/pom.xml | 2 +- hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml | 2 +- hapi-fhir-cli/pom.xml | 2 +- hapi-fhir-client-okhttp/pom.xml | 2 +- hapi-fhir-client/pom.xml | 2 +- hapi-fhir-converter/pom.xml | 2 +- hapi-fhir-dist/pom.xml | 2 +- hapi-fhir-docs/pom.xml | 2 +- ...1-bulk-export-stuck-in-finalize-state.yaml | 6 + hapi-fhir-jacoco/pom.xml | 2 +- hapi-fhir-jaxrsserver-base/pom.xml | 2 +- hapi-fhir-jpa/pom.xml | 2 +- hapi-fhir-jpaserver-base/pom.xml | 2 +- .../jpa/batch2/JpaJobPersistenceImpl.java | 11 +- .../dao/data/IBatch2WorkChunkRepository.java | 8 + .../pom.xml | 2 +- hapi-fhir-jpaserver-ips/pom.xml | 2 +- hapi-fhir-jpaserver-mdm/pom.xml | 2 +- hapi-fhir-jpaserver-model/pom.xml | 2 +- hapi-fhir-jpaserver-searchparam/pom.xml | 2 +- hapi-fhir-jpaserver-subscription/pom.xml | 2 +- hapi-fhir-jpaserver-test-dstu2/pom.xml | 2 +- hapi-fhir-jpaserver-test-dstu3/pom.xml | 2 +- hapi-fhir-jpaserver-test-r4/pom.xml | 2 +- .../uhn/fhir/jpa/bulk/BulkDataExportTest.java | 7 +- .../fhir/jpa/bulk/BulkExportUseCaseTest.java | 62 ++++++- hapi-fhir-jpaserver-test-r4b/pom.xml | 2 +- hapi-fhir-jpaserver-test-r5/pom.xml | 2 +- hapi-fhir-jpaserver-test-utilities/pom.xml | 2 +- hapi-fhir-jpaserver-uhnfhirtest/pom.xml | 2 +- hapi-fhir-server-mdm/pom.xml | 2 +- hapi-fhir-server-openapi/pom.xml | 2 +- hapi-fhir-server/pom.xml | 2 +- .../hapi-fhir-caching-api/pom.xml | 2 +- .../hapi-fhir-caching-caffeine/pom.xml | 4 +- .../hapi-fhir-caching-guava/pom.xml | 2 +- .../hapi-fhir-caching-testing/pom.xml | 2 +- hapi-fhir-serviceloaders/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../hapi-fhir-spring-boot-samples/pom.xml | 2 +- .../hapi-fhir-spring-boot-starter/pom.xml | 2 +- hapi-fhir-spring-boot/pom.xml | 2 +- hapi-fhir-sql-migrate/pom.xml | 2 +- hapi-fhir-storage-batch2-jobs/pom.xml | 2 +- hapi-fhir-storage-batch2/pom.xml | 2 +- .../uhn/fhir/batch2/api/IJobPersistence.java | 11 ++ .../fhir/batch2/config/BaseBatch2Config.java | 5 +- .../ReductionStepChunkProcessingResponse.java | 52 ++++++ .../coordinator/ReductionStepExecutor.java | 160 ++++++++++-------- .../SynchronizedJobPersistenceWrapper.java | 6 + .../coordinator/WorkChunkProcessor.java | 6 +- .../uhn/fhir/batch2/model/ChunkOutcome.java | 2 +- .../coordinator/JobCoordinatorImplTest.java | 6 +- .../coordinator/WorkChunkProcessorTest.java | 28 +-- hapi-fhir-storage-cr/pom.xml | 2 +- hapi-fhir-storage-mdm/pom.xml | 2 +- hapi-fhir-storage-test-utilities/pom.xml | 2 +- hapi-fhir-storage/pom.xml | 2 +- .../ca/uhn/fhir/jpa/api/config/DaoConfig.java | 4 +- hapi-fhir-structures-dstu2.1/pom.xml | 2 +- hapi-fhir-structures-dstu2/pom.xml | 2 +- hapi-fhir-structures-dstu3/pom.xml | 2 +- hapi-fhir-structures-hl7org-dstu2/pom.xml | 2 +- hapi-fhir-structures-r4/pom.xml | 2 +- hapi-fhir-structures-r4b/pom.xml | 2 +- hapi-fhir-structures-r5/pom.xml | 2 +- hapi-fhir-test-utilities/pom.xml | 2 +- hapi-fhir-testpage-overlay/pom.xml | 2 +- .../pom.xml | 2 +- hapi-fhir-validation-resources-dstu2/pom.xml | 2 +- hapi-fhir-validation-resources-dstu3/pom.xml | 2 +- hapi-fhir-validation-resources-r4/pom.xml | 2 +- hapi-fhir-validation-resources-r5/pom.xml | 2 +- hapi-fhir-validation/pom.xml | 2 +- hapi-tinder-plugin/pom.xml | 2 +- hapi-tinder-test/pom.xml | 2 +- pom.xml | 4 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- 89 files changed, 352 insertions(+), 176 deletions(-) create mode 100644 hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4511-bulk-export-stuck-in-finalize-state.yaml create mode 100644 hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepChunkProcessingResponse.java diff --git a/hapi-deployable-pom/pom.xml b/hapi-deployable-pom/pom.xml index 8cb99220fea..82d9cf0a407 100644 --- a/hapi-deployable-pom/pom.xml +++ b/hapi-deployable-pom/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-android/pom.xml b/hapi-fhir-android/pom.xml index 6526e2d844f..29f766e9479 100644 --- a/hapi-fhir-android/pom.xml +++ b/hapi-fhir-android/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-base/pom.xml b/hapi-fhir-base/pom.xml index 269d7037bce..d63977f228b 100644 --- a/hapi-fhir-base/pom.xml +++ b/hapi-fhir-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-bom/pom.xml b/hapi-fhir-bom/pom.xml index 80f0465f630..25fbc1f65c0 100644 --- a/hapi-fhir-bom/pom.xml +++ b/hapi-fhir-bom/pom.xml @@ -4,14 +4,14 @@ 4.0.0 ca.uhn.hapi.fhir hapi-fhir-bom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT pom HAPI FHIR BOM ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-checkstyle/pom.xml b/hapi-fhir-checkstyle/pom.xml index 12b5e688a7b..222581b8b6e 100644 --- a/hapi-fhir-checkstyle/pom.xml +++ b/hapi-fhir-checkstyle/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml index 0df7eabbc6c..916a44a92b0 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-api/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml index 2fcf899b565..bfc3fea4aca 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-app/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir-cli - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml index 36923a20b4f..3e72aca8bd2 100644 --- a/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml +++ b/hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../hapi-deployable-pom diff --git a/hapi-fhir-cli/pom.xml b/hapi-fhir-cli/pom.xml index 50f4c55da96..483750b34a9 100644 --- a/hapi-fhir-cli/pom.xml +++ b/hapi-fhir-cli/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-client-okhttp/pom.xml b/hapi-fhir-client-okhttp/pom.xml index 120dd6f69a0..9a2ac5e564a 100644 --- a/hapi-fhir-client-okhttp/pom.xml +++ b/hapi-fhir-client-okhttp/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-client/pom.xml b/hapi-fhir-client/pom.xml index 65e6d79aa63..f47b0576a9b 100644 --- a/hapi-fhir-client/pom.xml +++ b/hapi-fhir-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-converter/pom.xml b/hapi-fhir-converter/pom.xml index 7f2f73fc80e..6d5fb2b46d1 100644 --- a/hapi-fhir-converter/pom.xml +++ b/hapi-fhir-converter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-dist/pom.xml b/hapi-fhir-dist/pom.xml index f8f10fafeb0..ce23dae8540 100644 --- a/hapi-fhir-dist/pom.xml +++ b/hapi-fhir-dist/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-docs/pom.xml b/hapi-fhir-docs/pom.xml index f90af5cb5fc..4c2ed4698d7 100644 --- a/hapi-fhir-docs/pom.xml +++ b/hapi-fhir-docs/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4511-bulk-export-stuck-in-finalize-state.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4511-bulk-export-stuck-in-finalize-state.yaml new file mode 100644 index 00000000000..49003eff9ae --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4511-bulk-export-stuck-in-finalize-state.yaml @@ -0,0 +1,6 @@ +--- +type: fix +issue: 4511 +jira: SMILE-6064 +title: "Previously, bulk export jobs were getting stuck in the `FINALIZE` state when performed +with many resources and a low Bulk Export File Maximum Capacity. This has been fixed." diff --git a/hapi-fhir-jacoco/pom.xml b/hapi-fhir-jacoco/pom.xml index 9494c835c31..e4ac4a13a34 100644 --- a/hapi-fhir-jacoco/pom.xml +++ b/hapi-fhir-jacoco/pom.xml @@ -11,7 +11,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jaxrsserver-base/pom.xml b/hapi-fhir-jaxrsserver-base/pom.xml index 8ed093981c4..950d8c911c0 100644 --- a/hapi-fhir-jaxrsserver-base/pom.xml +++ b/hapi-fhir-jaxrsserver-base/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpa/pom.xml b/hapi-fhir-jpa/pom.xml index d5630cd4363..83264e1382b 100644 --- a/hapi-fhir-jpa/pom.xml +++ b/hapi-fhir-jpa/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index 44c5e67b405..511ea9a164d 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java index e3d16e5ab75..92dd356f0d6 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java @@ -29,13 +29,13 @@ import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; -import ca.uhn.fhir.util.Logs; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; import ca.uhn.fhir.jpa.util.JobInstanceUtil; import ca.uhn.fhir.model.api.PagingIterator; +import ca.uhn.fhir.util.Logs; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; @@ -59,6 +59,7 @@ import java.util.Set; import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -332,13 +333,21 @@ public class JpaJobPersistenceImpl implements IJobPersistence { } /** + * Deprecated, use {@link ca.uhn.fhir.jpa.batch2.JpaJobPersistenceImpl#fetchAllWorkChunksForStepStream(String, String)} * Note: Not @Transactional because the transaction happens in a lambda that's called outside of this method's scope */ @Override + @Deprecated public Iterator fetchAllWorkChunksForStepIterator(String theInstanceId, String theStepId) { return new PagingIterator<>((thePageIndex, theBatchSize, theConsumer) -> fetchChunksForStep(theInstanceId, theStepId, theBatchSize, thePageIndex, theConsumer)); } + @Override + @Transactional(propagation = Propagation.MANDATORY) + public Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId) { + return myWorkChunkRepository.fetchChunksForStep(theInstanceId, theStepId).map((entity) -> toChunk(entity, true)); + } + /** * Update the stored instance * diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java index a72ccd1cf79..16b68d6090a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java @@ -30,6 +30,7 @@ import org.springframework.data.repository.query.Param; import java.util.Date; import java.util.List; +import java.util.stream.Stream; public interface IBatch2WorkChunkRepository extends JpaRepository, IHapiFhirJpaRepository { @@ -39,9 +40,16 @@ public interface IBatch2WorkChunkRepository extends JpaRepository getDistinctStatusesForStep(@Param("instanceId") String theInstanceId, @Param("stepId") String theStepId); + /** + * Deprecated, use {@link ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository#fetchChunksForStep(String, String)} + */ + @Deprecated @Query("SELECT e FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId AND e.myTargetStepId = :targetStepId ORDER BY e.mySequence ASC") List fetchChunksForStep(Pageable thePageRequest, @Param("instanceId") String theInstanceId, @Param("targetStepId") String theTargetStepId); + @Query("SELECT e FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId AND e.myTargetStepId = :targetStepId ORDER BY e.mySequence ASC") + Stream fetchChunksForStep(@Param("instanceId") String theInstanceId, @Param("targetStepId") String theTargetStepId); + @Modifying @Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, e.myRecordsProcessed = :rp, e.mySerializedData = null WHERE e.myId = :id") void updateChunkStatusAndClearDataForEndSuccess(@Param("id") String theChunkId, @Param("et") Date theEndTime, @Param("rp") int theRecordsProcessed, @Param("status") StatusEnum theInProgress); diff --git a/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml b/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml index 6c6b3e6c5f6..1296ab631d8 100644 --- a/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml +++ b/hapi-fhir-jpaserver-elastic-test-utilities/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-ips/pom.xml b/hapi-fhir-jpaserver-ips/pom.xml index e52aea6b752..2068c8aabc4 100644 --- a/hapi-fhir-jpaserver-ips/pom.xml +++ b/hapi-fhir-jpaserver-ips/pom.xml @@ -3,7 +3,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-mdm/pom.xml b/hapi-fhir-jpaserver-mdm/pom.xml index 15aa9542fa4..119514c2bc9 100644 --- a/hapi-fhir-jpaserver-mdm/pom.xml +++ b/hapi-fhir-jpaserver-mdm/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-model/pom.xml b/hapi-fhir-jpaserver-model/pom.xml index a105674e4fa..6dbd6463b6e 100644 --- a/hapi-fhir-jpaserver-model/pom.xml +++ b/hapi-fhir-jpaserver-model/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-searchparam/pom.xml b/hapi-fhir-jpaserver-searchparam/pom.xml index 119880cd395..f4d1ac894d1 100755 --- a/hapi-fhir-jpaserver-searchparam/pom.xml +++ b/hapi-fhir-jpaserver-searchparam/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-subscription/pom.xml b/hapi-fhir-jpaserver-subscription/pom.xml index 8dd6082afeb..89ed657a078 100644 --- a/hapi-fhir-jpaserver-subscription/pom.xml +++ b/hapi-fhir-jpaserver-subscription/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-dstu2/pom.xml b/hapi-fhir-jpaserver-test-dstu2/pom.xml index 1bdf83b72ee..a50ebe2e721 100644 --- a/hapi-fhir-jpaserver-test-dstu2/pom.xml +++ b/hapi-fhir-jpaserver-test-dstu2/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-dstu3/pom.xml b/hapi-fhir-jpaserver-test-dstu3/pom.xml index f5797e922c1..93c7cd37d7f 100644 --- a/hapi-fhir-jpaserver-test-dstu3/pom.xml +++ b/hapi-fhir-jpaserver-test-dstu3/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-r4/pom.xml b/hapi-fhir-jpaserver-test-r4/pom.xml index 28e74fa348d..e7ef2570279 100644 --- a/hapi-fhir-jpaserver-test-r4/pom.xml +++ b/hapi-fhir-jpaserver-test-r4/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java index edc5aea249e..9038098aba2 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataExportTest.java @@ -49,6 +49,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -667,9 +668,11 @@ public class BulkDataExportTest extends BaseResourceProviderR4Test { assertNotNull(startResponse); // Run a scheduled pass to build the export - myBatch2JobHelper.awaitJobCompletion(startResponse.getJobId()); + myBatch2JobHelper.awaitJobCompletion(startResponse.getJobId(), 60); - await().until(() -> myJobRunner.getJobInfo(startResponse.getJobId()).getReport() != null); + await() + .atMost(120, TimeUnit.SECONDS) + .until(() -> myJobRunner.getJobInfo(startResponse.getJobId()).getReport() != null); // Iterate over the files String report = myJobRunner.getJobInfo(startResponse.getJobId()).getReport(); diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java index d510c26665d..89bd2e9acd0 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java @@ -6,11 +6,15 @@ import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.model.BulkExportJobResults; import ca.uhn.fhir.jpa.api.svc.IBatch2JobRunner; import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; +import ca.uhn.fhir.jpa.bulk.export.model.BulkExportJobStatusEnum; import ca.uhn.fhir.jpa.bulk.export.model.BulkExportResponseJson; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; +import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.util.BulkExportUtils; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.api.Constants; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions; import ca.uhn.fhir.util.JsonUtil; import ca.uhn.fhir.util.SearchParameterUtil; @@ -43,7 +47,7 @@ import org.springframework.beans.factory.annotation.Autowired; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -51,8 +55,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.awaitility.Awaitility.await; @@ -62,12 +66,12 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class BulkExportUseCaseTest extends BaseResourceProviderR4Test { @@ -406,6 +410,7 @@ public class BulkExportUseCaseTest extends BaseResourceProviderR4Test { @AfterEach public void after() { myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED); + myDaoConfig.setBulkExportFileMaximumCapacity(DaoConfig.DEFAULT_BULK_EXPORT_FILE_MAXIMUM_CAPACITY); } @Test @@ -433,6 +438,57 @@ public class BulkExportUseCaseTest extends BaseResourceProviderR4Test { assertThat(typeToContents.get("Observation"), containsString("obs-included")); assertThat(typeToContents.get("Observation"), not(containsString("obs-excluded"))); } + + @Test + public void testBulkExportWithLowMaxFileCapacity() { + final int numPatients = 250; + myDaoConfig.setBulkExportFileMaximumCapacity(1); + myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.ENABLED); + + RequestDetails details = new SystemRequestDetails(); + List patientIds = new ArrayList<>(); + for(int i = 0; i < numPatients; i++){ + String id = "p-"+i; + Patient patient = new Patient(); + patient.setId(id); + myPatientDao.update(patient, details); + patientIds.add(id); + } + + int patientsCreated = myPatientDao.search(SearchParameterMap.newSynchronous(), details).size(); + assertEquals(numPatients, patientsCreated); + + BulkDataExportOptions options = new BulkDataExportOptions(); + options.setResourceTypes(Sets.newHashSet("Patient")); + options.setExportStyle(BulkDataExportOptions.ExportStyle.PATIENT); + options.setOutputFormat(Constants.CT_FHIR_NDJSON); + + Batch2JobStartResponse job = myJobRunner.startNewJob(BulkExportUtils.createBulkExportJobParametersFromExportOptions(options)); + myBatch2JobHelper.awaitJobCompletion(job.getJobId(), 60); + ourLog.debug("Job status after awaiting - {}", myJobRunner.getJobInfo(job.getJobId()).getStatus()); + await() + .atMost(300, TimeUnit.SECONDS) + .until(() -> { + BulkExportJobStatusEnum status = myJobRunner.getJobInfo(job.getJobId()).getStatus(); + if (!BulkExportJobStatusEnum.COMPLETE.equals(status)) { + fail("Job status was changed from COMPLETE to " + status); + } + return myJobRunner.getJobInfo(job.getJobId()).getReport() != null; + }); + + String report = myJobRunner.getJobInfo(job.getJobId()).getReport(); + BulkExportJobResults results = JsonUtil.deserialize(report, BulkExportJobResults.class); + List binaryUrls = results.getResourceTypeToBinaryIds().get("Patient"); + + IParser jsonParser = myFhirContext.newJsonParser(); + for(String url : binaryUrls){ + Binary binary = myClient.read().resource(Binary.class).withUrl(url).execute(); + assertEquals(Constants.CT_FHIR_NDJSON, binary.getContentType()); + String resourceContents = new String(binary.getContent(), Constants.CHARSET_UTF8); + String resourceId = jsonParser.parseResource(resourceContents).getIdElement().getIdPart(); + assertTrue(patientIds.contains(resourceId)); + } + } } diff --git a/hapi-fhir-jpaserver-test-r4b/pom.xml b/hapi-fhir-jpaserver-test-r4b/pom.xml index c11ae5a2df8..28371d35696 100644 --- a/hapi-fhir-jpaserver-test-r4b/pom.xml +++ b/hapi-fhir-jpaserver-test-r4b/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-r5/pom.xml b/hapi-fhir-jpaserver-test-r5/pom.xml index 7060c408d81..22576e796c4 100644 --- a/hapi-fhir-jpaserver-test-r5/pom.xml +++ b/hapi-fhir-jpaserver-test-r5/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-test-utilities/pom.xml b/hapi-fhir-jpaserver-test-utilities/pom.xml index fc01acda8a3..06e7765f1ef 100644 --- a/hapi-fhir-jpaserver-test-utilities/pom.xml +++ b/hapi-fhir-jpaserver-test-utilities/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml index 35754d1c594..8d41a6b044f 100644 --- a/hapi-fhir-jpaserver-uhnfhirtest/pom.xml +++ b/hapi-fhir-jpaserver-uhnfhirtest/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-server-mdm/pom.xml b/hapi-fhir-server-mdm/pom.xml index 244a4e150be..3e35af08e36 100644 --- a/hapi-fhir-server-mdm/pom.xml +++ b/hapi-fhir-server-mdm/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server-openapi/pom.xml b/hapi-fhir-server-openapi/pom.xml index 09d5cfc1f09..5b72f14bacb 100644 --- a/hapi-fhir-server-openapi/pom.xml +++ b/hapi-fhir-server-openapi/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-server/pom.xml b/hapi-fhir-server/pom.xml index d4b4abe54dd..7512da84996 100644 --- a/hapi-fhir-server/pom.xml +++ b/hapi-fhir-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml index 3a198fe53b6..51a9fd45e8c 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-api/pom.xml @@ -7,7 +7,7 @@ hapi-fhir-serviceloaders ca.uhn.hapi.fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml index 3db8187c54d..c19e1042777 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-caffeine/pom.xml @@ -7,7 +7,7 @@ hapi-fhir-serviceloaders ca.uhn.hapi.fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml @@ -20,7 +20,7 @@ ca.uhn.hapi.fhir hapi-fhir-caching-api - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT com.github.ben-manes.caffeine diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml index e58f3afc9ff..cb040e4c501 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-guava/pom.xml @@ -7,7 +7,7 @@ hapi-fhir-serviceloaders ca.uhn.hapi.fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml b/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml index 8a320dedd4b..938b2d2a076 100644 --- a/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml +++ b/hapi-fhir-serviceloaders/hapi-fhir-caching-testing/pom.xml @@ -7,7 +7,7 @@ hapi-fhir ca.uhn.hapi.fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../pom.xml diff --git a/hapi-fhir-serviceloaders/pom.xml b/hapi-fhir-serviceloaders/pom.xml index bc7ea7deba4..c298864c2d6 100644 --- a/hapi-fhir-serviceloaders/pom.xml +++ b/hapi-fhir-serviceloaders/pom.xml @@ -5,7 +5,7 @@ hapi-fhir ca.uhn.hapi.fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml index 8b6fdd97fec..19feb37f903 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-autoconfigure/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml index ea7b07a4ccc..40e1c38ddf1 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-apache/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT hapi-fhir-spring-boot-sample-client-apache diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml index 4f6393f7c0b..df208d1bf08 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-client-okhttp/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT hapi-fhir-spring-boot-sample-client-okhttp diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml index 3ec1440cb18..f4dc2889b56 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/hapi-fhir-spring-boot-sample-server-jersey/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot-samples - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT hapi-fhir-spring-boot-sample-server-jersey diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml index 92e14112893..3ef3126c82e 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-samples/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir-spring-boot - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT hapi-fhir-spring-boot-samples diff --git a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml index d3a530ce6a0..28418fc4ddf 100644 --- a/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml +++ b/hapi-fhir-spring-boot/hapi-fhir-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-spring-boot/pom.xml b/hapi-fhir-spring-boot/pom.xml index aafd16791fe..0d9a4d04435 100644 --- a/hapi-fhir-spring-boot/pom.xml +++ b/hapi-fhir-spring-boot/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-sql-migrate/pom.xml b/hapi-fhir-sql-migrate/pom.xml index 7f465bd3e06..81220029411 100644 --- a/hapi-fhir-sql-migrate/pom.xml +++ b/hapi-fhir-sql-migrate/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-batch2-jobs/pom.xml b/hapi-fhir-storage-batch2-jobs/pom.xml index e2e613dfadf..286a5d11aee 100644 --- a/hapi-fhir-storage-batch2-jobs/pom.xml +++ b/hapi-fhir-storage-batch2-jobs/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-storage-batch2/pom.xml b/hapi-fhir-storage-batch2/pom.xml index dd334e97348..b4c2a62f251 100644 --- a/hapi-fhir-storage-batch2/pom.xml +++ b/hapi-fhir-storage-batch2/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java index a8801cf8f7b..b4a4d838ad5 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java @@ -38,6 +38,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; public interface IJobPersistence { @@ -196,14 +197,24 @@ public interface IJobPersistence { Iterator fetchAllWorkChunksIterator(String theInstanceId, boolean theWithData); /** + * Deprecated, use {@link ca.uhn.fhir.batch2.api.IJobPersistence#fetchAllWorkChunksForStepStream(String, String)} * Fetch all chunks with data for a given instance for a given step id * @param theInstanceId * @param theStepId * @return - an iterator for fetching work chunks */ + @Deprecated Iterator fetchAllWorkChunksForStepIterator(String theInstanceId, String theStepId); + /** + * Fetch all chunks with data for a given instance for a given step id + * @param theInstanceId + * @param theStepId + * @return - a stream for fetching work chunks + */ + Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId); + /** * Update the stored instance. If the status is changing, use {@link ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater} * instead to ensure state-change callbacks are invoked properly. diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java index 41bfa35d00f..a781bb75f6b 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java @@ -39,6 +39,7 @@ import ca.uhn.fhir.jpa.subscription.channel.api.IChannelReceiver; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; @Configuration public abstract class BaseBatch2Config { @@ -56,8 +57,8 @@ public abstract class BaseBatch2Config { } @Bean - public WorkChunkProcessor jobStepExecutorService(BatchJobSender theBatchJobSender) { - return new WorkChunkProcessor(myPersistence, theBatchJobSender); + public WorkChunkProcessor jobStepExecutorService(BatchJobSender theBatchJobSender, PlatformTransactionManager theTransactionManager) { + return new WorkChunkProcessor(myPersistence, theBatchJobSender, theTransactionManager); } @Bean diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepChunkProcessingResponse.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepChunkProcessingResponse.java new file mode 100644 index 00000000000..34d4c60b0ff --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepChunkProcessingResponse.java @@ -0,0 +1,52 @@ +package ca.uhn.fhir.batch2.coordinator; + +import ca.uhn.fhir.batch2.model.WorkChunk; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +public class ReductionStepChunkProcessingResponse { + + private List mySuccessfulChunkIds; + private List myFailedChunksIds; + private boolean myIsSuccessful; + + public ReductionStepChunkProcessingResponse(boolean theDefaultSuccessValue){ + mySuccessfulChunkIds = new ArrayList<>(); + myFailedChunksIds = new ArrayList<>(); + myIsSuccessful = theDefaultSuccessValue; + } + + public List getSuccessfulChunkIds() { + return mySuccessfulChunkIds; + } + + public boolean hasSuccessfulChunksIds(){ + return !CollectionUtils.isEmpty(mySuccessfulChunkIds); + } + + public void addSuccessfulChunkId(WorkChunk theWorkChunk){ + mySuccessfulChunkIds.add(theWorkChunk.getId()); + } + + public List getFailedChunksIds() { + return myFailedChunksIds; + } + + public boolean hasFailedChunkIds(){ + return !CollectionUtils.isEmpty(myFailedChunksIds); + } + + public void addFailedChunkId(WorkChunk theWorChunk){ + myFailedChunksIds.add(theWorChunk.getId()); + } + + public boolean isSuccessful(){ + return myIsSuccessful; + } + + public void setSuccessful(boolean theSuccessValue){ + myIsSuccessful = theSuccessValue; + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutor.java index 463109a2e14..4b4a88dbc74 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutor.java @@ -31,17 +31,23 @@ import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.Logs; import org.slf4j.Logger; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.stream.Stream; public class ReductionStepExecutor { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); private final IJobPersistence myJobPersistence; + private final PlatformTransactionManager myTxManager; + private final TransactionTemplate myTxTemplate; - public ReductionStepExecutor(IJobPersistence theJobPersistence) { + public ReductionStepExecutor(IJobPersistence theJobPersistence, PlatformTransactionManager theTransactionManager) { myJobPersistence = theJobPersistence; + myTxManager = theTransactionManager; + myTxTemplate = new TransactionTemplate(theTransactionManager); + myTxTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); } /** @@ -67,86 +73,32 @@ public class ReductionStepExecutor { } theInstance.setStatus(StatusEnum.FINALIZE); - // We fetch all chunks first... - Iterator chunkIterator = myJobPersistence.fetchAllWorkChunksForStepIterator(theInstance.getInstanceId(), theStep.getStepId()); - - List failedChunks = new ArrayList<>(); - List successfulChunkIds = new ArrayList<>(); - - boolean retval = true; + boolean defaultSuccessValue = true; + ReductionStepChunkProcessingResponse response = new ReductionStepChunkProcessingResponse(defaultSuccessValue); try { - while (chunkIterator.hasNext()) { - WorkChunk chunk = chunkIterator.next(); - if (!chunk.getStatus().isIncomplete()) { - // This should never happen since jobs with reduction are required to be gated - ourLog.error("Unexpected chunk {} with status {} found while reducing {}. No chunks feeding into a reduction step should be complete.", chunk.getId(), chunk.getStatus(), theInstance); - continue; + myTxTemplate.executeWithoutResult((status) -> { + try(Stream chunkIterator2 = myJobPersistence.fetchAllWorkChunksForStepStream(theInstance.getInstanceId(), theStep.getStepId())) { + chunkIterator2.forEach((chunk) -> { + processChunk(chunk, theInstance, theInputType, theParameters, reductionStepWorker, response); + }); } - - if (!failedChunks.isEmpty()) { - // we are going to fail all future chunks now - failedChunks.add(chunk.getId()); - } else { - try { - // feed them into our reduction worker - // this is the most likely area to throw, - // as this is where db actions and processing is likely to happen - ChunkExecutionDetails chunkDetails = new ChunkExecutionDetails<>(chunk.getData(theInputType), theParameters, theInstance.getInstanceId(), chunk.getId()); - - ChunkOutcome outcome = reductionStepWorker.consume(chunkDetails); - - switch (outcome.getStatuss()) { - case SUCCESS: - successfulChunkIds.add(chunk.getId()); - break; - - case ABORT: - ourLog.error("Processing of work chunk {} resulted in aborting job.", chunk.getId()); - - // fail entire job - including all future workchunks - failedChunks.add(chunk.getId()); - retval = false; - break; - - case FAIL: - // non-idempotent; but failed chunks will be - // ignored on a second runthrough of reduction step - myJobPersistence.markWorkChunkAsFailed(chunk.getId(), - "Step worker failed to process work chunk " + chunk.getId()); - retval = false; - break; - } - } catch (Exception e) { - String msg = String.format( - "Reduction step failed to execute chunk reduction for chunk %s with exception: %s.", - chunk.getId(), - e.getMessage() - ); - // we got a failure in a reduction - ourLog.error(msg, e); - retval = false; - - myJobPersistence.markWorkChunkAsFailed(chunk.getId(), msg); - } - } - } - + }); } finally { - if (!successfulChunkIds.isEmpty()) { + if (response.hasSuccessfulChunksIds()) { // complete the steps without making a new work chunk myJobPersistence.markWorkChunksWithStatusAndWipeData(theInstance.getInstanceId(), - successfulChunkIds, + response.getSuccessfulChunkIds(), StatusEnum.COMPLETED, null // error message - none ); } - if (!failedChunks.isEmpty()) { + if (response.hasFailedChunkIds()) { // mark any failed chunks as failed for aborting myJobPersistence.markWorkChunksWithStatusAndWipeData(theInstance.getInstanceId(), - failedChunks, + response.getFailedChunksIds(), StatusEnum.FAILED, "JOB ABORTED"); } @@ -154,10 +106,72 @@ public class ReductionStepExecutor { } // if no successful chunks, return false - if (successfulChunkIds.isEmpty()) { - retval = false; + if (!response.hasSuccessfulChunksIds()) { + response.setSuccessful(false); } - return retval; + return response.isSuccessful(); + } + + private + void processChunk(WorkChunk theChunk, + JobInstance theInstance, + Class theInputType, + PT theParameters, + IReductionStepWorker theReductionStepWorker, + ReductionStepChunkProcessingResponse theResponseObject){ + + if (!theChunk.getStatus().isIncomplete()) { + // This should never happen since jobs with reduction are required to be gated + ourLog.error("Unexpected chunk {} with status {} found while reducing {}. No chunks feeding into a reduction step should be complete.", theChunk.getId(), theChunk.getStatus(), theInstance); + return; + } + + if (theResponseObject.hasFailedChunkIds()) { + // we are going to fail all future chunks now + theResponseObject.addFailedChunkId(theChunk); + } else { + try { + // feed them into our reduction worker + // this is the most likely area to throw, + // as this is where db actions and processing is likely to happen + ChunkExecutionDetails chunkDetails = new ChunkExecutionDetails<>(theChunk.getData(theInputType), theParameters, theInstance.getInstanceId(), theChunk.getId()); + + ChunkOutcome outcome = theReductionStepWorker.consume(chunkDetails); + + switch (outcome.getStatus()) { + case SUCCESS: + theResponseObject.addSuccessfulChunkId(theChunk); + break; + + case ABORT: + ourLog.error("Processing of work chunk {} resulted in aborting job.", theChunk.getId()); + + // fail entire job - including all future workchunks + theResponseObject.addFailedChunkId(theChunk); + theResponseObject.setSuccessful(false); + break; + + case FAIL: + // non-idempotent; but failed chunks will be + // ignored on a second runthrough of reduction step + myJobPersistence.markWorkChunkAsFailed(theChunk.getId(), + "Step worker failed to process work chunk " + theChunk.getId()); + theResponseObject.setSuccessful(false); + break; + } + } catch (Exception e) { + String msg = String.format( + "Reduction step failed to execute chunk reduction for chunk %s with exception: %s.", + theChunk.getId(), + e.getMessage() + ); + // we got a failure in a reduction + ourLog.error(msg, e); + theResponseObject.setSuccessful(false); + + myJobPersistence.markWorkChunkAsFailed(theChunk.getId(), msg); + } + } } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/SynchronizedJobPersistenceWrapper.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/SynchronizedJobPersistenceWrapper.java index b26aaa82451..7bceba6629e 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/SynchronizedJobPersistenceWrapper.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/SynchronizedJobPersistenceWrapper.java @@ -36,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; public class SynchronizedJobPersistenceWrapper implements IJobPersistence { @@ -153,6 +154,11 @@ public class SynchronizedJobPersistenceWrapper implements IJobPersistence { return myWrap.fetchAllWorkChunksForStepIterator(theInstanceId, theStepId); } + @Override + public Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId) { + return myWrap.fetchAllWorkChunksForStepStream(theInstanceId, theStepId); + } + @Override public synchronized boolean updateInstance(JobInstance theInstance) { return myWrap.updateInstance(theInstance); diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java index 8dba0b14492..4ba661a58fe 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java @@ -36,6 +36,7 @@ import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.Logs; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; +import org.springframework.transaction.PlatformTransactionManager; import javax.annotation.Nullable; import java.util.Optional; @@ -63,11 +64,12 @@ public class WorkChunkProcessor { private final ReductionStepExecutor myReductionStepExecutor; public WorkChunkProcessor(IJobPersistence theJobPersistence, - BatchJobSender theSender) { + BatchJobSender theSender, + PlatformTransactionManager theTransactionManager) { myJobPersistence = theJobPersistence; myBatchJobSender = theSender; myStepExecutor = new StepExecutor(theJobPersistence); - myReductionStepExecutor = new ReductionStepExecutor(theJobPersistence); + myReductionStepExecutor = new ReductionStepExecutor(theJobPersistence, theTransactionManager); } /** diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/ChunkOutcome.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/ChunkOutcome.java index 963c4c43073..20fd16c21f8 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/ChunkOutcome.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/ChunkOutcome.java @@ -33,7 +33,7 @@ public class ChunkOutcome { myStatus = theStatus; } - public Status getStatuss() { + public Status getStatus() { return myStatus; } diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java index dc6aa312e4b..d5a344ddf36 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java @@ -36,6 +36,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.stubbing.Answer; import org.springframework.messaging.MessageDeliveryException; +import org.springframework.transaction.PlatformTransactionManager; import javax.annotation.Nonnull; import java.util.Arrays; @@ -43,7 +44,6 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -69,6 +69,8 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { private JobDefinitionRegistry myJobDefinitionRegistry; @Mock private IJobMaintenanceService myJobMaintenanceService; + @Mock + private PlatformTransactionManager myPlatformTransactionManager; @Captor private ArgumentCaptor> myStep1ExecutionDetailsCaptor; @@ -87,7 +89,7 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { public void beforeEach() { // The code refactored to keep the same functionality, // but in this service (so it's a real service here!) - WorkChunkProcessor jobStepExecutorSvc = new WorkChunkProcessor(myJobInstancePersister, myBatchJobSender); + WorkChunkProcessor jobStepExecutorSvc = new WorkChunkProcessor(myJobInstancePersister, myBatchJobSender, myPlatformTransactionManager); mySvc = new JobCoordinatorImpl(myBatchJobSender, myWorkChannelReceiver, myJobInstancePersister, myJobDefinitionRegistry, jobStepExecutorSvc, myJobMaintenanceService); } diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java index 2c6a59e4d64..78c66f88666 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.transaction.PlatformTransactionManager; import java.util.ArrayList; import java.util.Arrays; @@ -104,8 +105,8 @@ public class WorkChunkProcessorTest { // our test class private class TestWorkChunkProcessor extends WorkChunkProcessor { - public TestWorkChunkProcessor(IJobPersistence thePersistence, BatchJobSender theSender) { - super(thePersistence, theSender); + public TestWorkChunkProcessor(IJobPersistence thePersistence, BatchJobSender theSender, PlatformTransactionManager theTransactionManager) { + super(thePersistence, theSender, theTransactionManager); } @Override @@ -138,11 +139,14 @@ public class WorkChunkProcessorTest { @Mock private BatchJobSender myJobSender; + @Mock + private PlatformTransactionManager myMockTransactionManager; + private TestWorkChunkProcessor myExecutorSvc; @BeforeEach public void init() { - myExecutorSvc = new TestWorkChunkProcessor(myJobPersistence, myJobSender); + myExecutorSvc = new TestWorkChunkProcessor(myJobPersistence, myJobSender, myMockTransactionManager); } private JobDefinitionStep mockOutWorkCursor( @@ -197,8 +201,8 @@ public class WorkChunkProcessorTest { // when when(workCursor.isReductionStep()) .thenReturn(true); - when(myJobPersistence.fetchAllWorkChunksForStepIterator(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) - .thenReturn(chunks.iterator()); + when(myJobPersistence.fetchAllWorkChunksForStepStream(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) + .thenReturn(chunks.stream()); when(myJobPersistence.markInstanceAsStatus(eq(INSTANCE_ID), eq(StatusEnum.FINALIZE))).thenReturn(true); when(myReductionStep.consume(any(ChunkExecutionDetails.class))) .thenReturn(ChunkOutcome.SUCCESS()); @@ -259,8 +263,8 @@ public class WorkChunkProcessorTest { // when when(workCursor.isReductionStep()) .thenReturn(true); - when(myJobPersistence.fetchAllWorkChunksForStepIterator(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) - .thenReturn(chunks.iterator()); + when(myJobPersistence.fetchAllWorkChunksForStepStream(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) + .thenReturn(chunks.stream()); when(myJobPersistence.markInstanceAsStatus(eq(INSTANCE_ID), eq(StatusEnum.FINALIZE))).thenReturn(true); doThrow(new RuntimeException(errorMsg)) .when(myReductionStep).consume(any(ChunkExecutionDetails.class)); @@ -308,8 +312,8 @@ public class WorkChunkProcessorTest { // when when(workCursor.isReductionStep()) .thenReturn(true); - when(myJobPersistence.fetchAllWorkChunksForStepIterator(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) - .thenReturn(chunks.iterator()); + when(myJobPersistence.fetchAllWorkChunksForStepStream(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) + .thenReturn(chunks.stream()); when(myJobPersistence.markInstanceAsStatus(eq(INSTANCE_ID), eq(StatusEnum.FINALIZE))).thenReturn(true); when(myReductionStep.consume(any(ChunkExecutionDetails.class))) .thenReturn(ChunkOutcome.SUCCESS()) @@ -355,8 +359,8 @@ public class WorkChunkProcessorTest { when(workCursor.isReductionStep()) .thenReturn(true); when(myJobPersistence.markInstanceAsStatus(eq(INSTANCE_ID), eq(StatusEnum.FINALIZE))).thenReturn(true); - when(myJobPersistence.fetchAllWorkChunksForStepIterator(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) - .thenReturn(chunks.iterator()); + when(myJobPersistence.fetchAllWorkChunksForStepStream(eq(INSTANCE_ID), eq(REDUCTION_STEP_ID))) + .thenReturn(chunks.stream()); when(myReductionStep.consume(any(ChunkExecutionDetails.class))) .thenReturn(ChunkOutcome.SUCCESS()) .thenReturn(new ChunkOutcome(ChunkOutcome.Status.ABORT)); @@ -609,7 +613,7 @@ public class WorkChunkProcessorTest { verify(myJobPersistence, never()) .markWorkChunksWithStatusAndWipeData(anyString(), anyList(), any(), any()); verify(myJobPersistence, never()) - .fetchAllWorkChunksForStepIterator(anyString(), anyString()); + .fetchAllWorkChunksForStepStream(anyString(), anyString()); } private JobInstance getTestJobInstance() { diff --git a/hapi-fhir-storage-cr/pom.xml b/hapi-fhir-storage-cr/pom.xml index d526868807a..5145cc685df 100644 --- a/hapi-fhir-storage-cr/pom.xml +++ b/hapi-fhir-storage-cr/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage-mdm/pom.xml b/hapi-fhir-storage-mdm/pom.xml index a6398f0e0bf..68adb73cec2 100644 --- a/hapi-fhir-storage-mdm/pom.xml +++ b/hapi-fhir-storage-mdm/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-storage-test-utilities/pom.xml b/hapi-fhir-storage-test-utilities/pom.xml index 0418ffd4544..1c3b5b17eda 100644 --- a/hapi-fhir-storage-test-utilities/pom.xml +++ b/hapi-fhir-storage-test-utilities/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml 4.0.0 diff --git a/hapi-fhir-storage/pom.xml b/hapi-fhir-storage/pom.xml index eb487e974c3..f604a006dc6 100644 --- a/hapi-fhir-storage/pom.xml +++ b/hapi-fhir-storage/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java index d229b057ab6..7a534c007e9 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java @@ -97,6 +97,8 @@ public class DaoConfig { public static final int DEFAULT_BUNDLE_BATCH_POOL_SIZE = 20; // 1 for single thread public static final int DEFAULT_BUNDLE_BATCH_MAX_POOL_SIZE = 100; // 1 for single thread public static final int DEFAULT_BUNDLE_BATCH_QUEUE_CAPACITY = 200; + + public static final int DEFAULT_BULK_EXPORT_FILE_MAXIMUM_CAPACITY = 1_000; /** * Default value for {@link #setMaximumSearchResultCountInTransaction(Integer)} * @@ -332,7 +334,7 @@ public class DaoConfig { /** * Since 6.2.0 */ - private int myBulkExportFileMaximumCapacity = 1_000; + private int myBulkExportFileMaximumCapacity = DEFAULT_BULK_EXPORT_FILE_MAXIMUM_CAPACITY; /** * Since 6.4.0 */ diff --git a/hapi-fhir-structures-dstu2.1/pom.xml b/hapi-fhir-structures-dstu2.1/pom.xml index 104266a8abe..faa3ce789db 100644 --- a/hapi-fhir-structures-dstu2.1/pom.xml +++ b/hapi-fhir-structures-dstu2.1/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu2/pom.xml b/hapi-fhir-structures-dstu2/pom.xml index c80e11b4ab5..a2751df7f82 100644 --- a/hapi-fhir-structures-dstu2/pom.xml +++ b/hapi-fhir-structures-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-dstu3/pom.xml b/hapi-fhir-structures-dstu3/pom.xml index 1a28e61dcfd..17729087318 100644 --- a/hapi-fhir-structures-dstu3/pom.xml +++ b/hapi-fhir-structures-dstu3/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-hl7org-dstu2/pom.xml b/hapi-fhir-structures-hl7org-dstu2/pom.xml index c486671fc7c..4cf4f6a77af 100644 --- a/hapi-fhir-structures-hl7org-dstu2/pom.xml +++ b/hapi-fhir-structures-hl7org-dstu2/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4/pom.xml b/hapi-fhir-structures-r4/pom.xml index 01f7b6163e8..e7b3d652b67 100644 --- a/hapi-fhir-structures-r4/pom.xml +++ b/hapi-fhir-structures-r4/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r4b/pom.xml b/hapi-fhir-structures-r4b/pom.xml index 477572bda4a..7dd2b0e3102 100644 --- a/hapi-fhir-structures-r4b/pom.xml +++ b/hapi-fhir-structures-r4b/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-structures-r5/pom.xml b/hapi-fhir-structures-r5/pom.xml index b88b92ffb83..3af03cad003 100644 --- a/hapi-fhir-structures-r5/pom.xml +++ b/hapi-fhir-structures-r5/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-test-utilities/pom.xml b/hapi-fhir-test-utilities/pom.xml index de8a86a686e..28ac6e5c40d 100644 --- a/hapi-fhir-test-utilities/pom.xml +++ b/hapi-fhir-test-utilities/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-testpage-overlay/pom.xml b/hapi-fhir-testpage-overlay/pom.xml index 92298aefe31..005d0205b6d 100644 --- a/hapi-fhir-testpage-overlay/pom.xml +++ b/hapi-fhir-testpage-overlay/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-fhir-validation-resources-dstu2.1/pom.xml b/hapi-fhir-validation-resources-dstu2.1/pom.xml index 149494e2c09..c1bf35b88e0 100644 --- a/hapi-fhir-validation-resources-dstu2.1/pom.xml +++ b/hapi-fhir-validation-resources-dstu2.1/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu2/pom.xml b/hapi-fhir-validation-resources-dstu2/pom.xml index 9317ff0611d..d4d395c2381 100644 --- a/hapi-fhir-validation-resources-dstu2/pom.xml +++ b/hapi-fhir-validation-resources-dstu2/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-dstu3/pom.xml b/hapi-fhir-validation-resources-dstu3/pom.xml index 3ae7b8be7c5..3d74783f78f 100644 --- a/hapi-fhir-validation-resources-dstu3/pom.xml +++ b/hapi-fhir-validation-resources-dstu3/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r4/pom.xml b/hapi-fhir-validation-resources-r4/pom.xml index e372901ae19..66f101d2d79 100644 --- a/hapi-fhir-validation-resources-r4/pom.xml +++ b/hapi-fhir-validation-resources-r4/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation-resources-r5/pom.xml b/hapi-fhir-validation-resources-r5/pom.xml index 5fa0d7acaf3..5764a2b0a1d 100644 --- a/hapi-fhir-validation-resources-r5/pom.xml +++ b/hapi-fhir-validation-resources-r5/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-fhir-validation/pom.xml b/hapi-fhir-validation/pom.xml index 0012fe578e5..dfdd5391a31 100644 --- a/hapi-fhir-validation/pom.xml +++ b/hapi-fhir-validation/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-deployable-pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../hapi-deployable-pom/pom.xml diff --git a/hapi-tinder-plugin/pom.xml b/hapi-tinder-plugin/pom.xml index 259e7c43bbd..b172ccaf8a8 100644 --- a/hapi-tinder-plugin/pom.xml +++ b/hapi-tinder-plugin/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/hapi-tinder-test/pom.xml b/hapi-tinder-test/pom.xml index bd99b33ed39..83007e44de3 100644 --- a/hapi-tinder-test/pom.xml +++ b/hapi-tinder-test/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 2f3a7bcef3c..d44ec8f9301 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ ca.uhn.hapi.fhir hapi-fhir pom - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT HAPI-FHIR An open-source implementation of the FHIR specification in Java. https://hapifhir.io @@ -2140,7 +2140,7 @@ ca.uhn.hapi.fhir hapi-fhir-checkstyle - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT diff --git a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml index 9c449d93b38..09e9106912b 100644 --- a/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml +++ b/tests/hapi-fhir-base-test-jaxrsserver-kotlin/pom.xml @@ -6,7 +6,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-client/pom.xml b/tests/hapi-fhir-base-test-mindeps-client/pom.xml index 0dd0d834c18..a3f893dbf51 100644 --- a/tests/hapi-fhir-base-test-mindeps-client/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-client/pom.xml @@ -4,7 +4,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../pom.xml diff --git a/tests/hapi-fhir-base-test-mindeps-server/pom.xml b/tests/hapi-fhir-base-test-mindeps-server/pom.xml index 74ea4e9adb0..a6a52f1cc60 100644 --- a/tests/hapi-fhir-base-test-mindeps-server/pom.xml +++ b/tests/hapi-fhir-base-test-mindeps-server/pom.xml @@ -5,7 +5,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.3.14-SNAPSHOT + 6.3.15-SNAPSHOT ../../pom.xml